From d0f9aefb57e1efe549e8aa2855bde564b820eb03 Mon Sep 17 00:00:00 2001 From: Shreyas Telkar Date: Tue, 17 Nov 2026 15:04:41 -0800 Subject: [PATCH 01/34] feat: `finops_databricks_savings`: Databricks SQL warehouse savings - measures one row of usage facts per SQL warehouse of the connection's workspace (uptime, idle share, parallelism, spill, queueing, DBUs and list-price cost) from Databricks system tables, for the seven days to 06:00 UTC today, with the window end and warehouse id as bind values - altimate-core decides which warehouses qualify and what that saves (`databricksWarehouseSavings`); an older core says to update altimate-code - the tool gives Claude each finding with its evidence and monthly range, and the full report as metadata; registered for `tool run`, the agent's tool list and the analyst's read-only set - tests: tool output and metadata, failure paths, fact coercion, the bind order, and a live case in the Databricks e2e suite Co-Authored-By: Claude Opus 5.5 --- packages/opencode/src/agent/agent.ts | 1 + .../native/finops/databricks-savings.ts | 226 ++++++++++++++++++ .../src/altimate/native/finops/register.ts | 7 + .../opencode/src/altimate/native/types.ts | 44 ++++ .../src/altimate/tool-cli/allow-list.ts | 2 + .../tools/finops-databricks-savings.ts | 115 +++++++++ packages/opencode/src/tool/registry.ts | 2 + .../test/agent/analyst-data-explorer.test.ts | 1 + .../altimate/finops-databricks-e2e.test.ts | 15 ++ .../finops-databricks-savings.test.ts | 178 ++++++++++++++ 10 files changed, 591 insertions(+) create mode 100644 packages/opencode/src/altimate/native/finops/databricks-savings.ts create mode 100644 packages/opencode/src/altimate/tools/finops-databricks-savings.ts create mode 100644 packages/opencode/test/altimate/finops-databricks-savings.test.ts diff --git a/packages/opencode/src/agent/agent.ts b/packages/opencode/src/agent/agent.ts index 0884f50bbc..2ac7aec2a7 100644 --- a/packages/opencode/src/agent/agent.ts +++ b/packages/opencode/src/agent/agent.ts @@ -267,6 +267,7 @@ export const layer = Layer.effect( finops_analyze_credits: "allow", finops_expensive_queries: "allow", finops_warehouse_advice: "allow", + finops_databricks_savings: "allow", finops_unused_resources: "allow", finops_role_grants: "allow", finops_role_hierarchy: "allow", diff --git a/packages/opencode/src/altimate/native/finops/databricks-savings.ts b/packages/opencode/src/altimate/native/finops/databricks-savings.ts new file mode 100644 index 0000000000..e7e1a13774 --- /dev/null +++ b/packages/opencode/src/altimate/native/finops/databricks-savings.ts @@ -0,0 +1,226 @@ +// altimate_change - new file +/** + * Databricks SQL warehouse savings: measures one row of usage facts per SQL warehouse of + * the connection's workspace from Databricks system tables, and hands them to + * altimate-core, which decides which warehouses qualify and what that saves. The rules + * and prices live in core; this file only measures. + */ + +import * as core from "@altimateai/altimate-core" +import * as Registry from "../connections/registry" +import { resolveFinopsWarehouse } from "./warehouse-resolver" +import type { DatabricksSavingsParams, DatabricksSavingsResult, DatabricksSavingsReport } from "../types" + +const SUPPORTED_TYPES = ["databricks"] as const + +/** Days of usage the facts cover; the rules price a day of this window. */ +const LOOKBACK_DAYS = 7 + +const DAY_MS = 86_400_000 + +/** A SQL warehouse's id in a connection's HTTP path (`/sql/1.0/warehouses/`). */ +const WAREHOUSE_PATH = /\/warehouses\/([0-9a-z]+)\/?$/i + +/** + * Per-warehouse usage over the seven days to 06:00 UTC on the first parameter, for every + * warehouse in the workspace of the warehouse named by the second. System tables hold + * every workspace of the account, so each input is cut to that workspace's warehouses. + * Billing that landed after the window's end is left out, so a past window gives the + * same facts on every run. + */ +export const DATABRICKS_WAREHOUSE_FACTS_SQL = ` +WITH p AS ( + SELECT CAST(? AS DATE) AS we_date +), +w AS ( + SELECT we_date, + CAST(we_date AS TIMESTAMP) + INTERVAL 6 HOURS AS r, + CAST(we_date AS TIMESTAMP) + INTERVAL 6 HOURS - INTERVAL 7 DAYS AS rs + FROM p +), +inv AS ( + SELECT workspace_id, warehouse_id, warehouse_name, warehouse_type, warehouse_size, delete_time + FROM system.compute.warehouses + WHERE workspace_id = (SELECT MAX(workspace_id) FROM system.compute.warehouses WHERE warehouse_id = ?) + QUALIFY ROW_NUMBER() OVER (PARTITION BY workspace_id, warehouse_id ORDER BY change_time DESC) = 1 +), +ev AS ( + SELECT e.workspace_id, e.warehouse_id, e.event_type, e.event_time, e.cluster_count, + unix_timestamp(LEAD(e.event_time) OVER (PARTITION BY e.workspace_id, e.warehouse_id ORDER BY e.event_time)) + - unix_timestamp(e.event_time) AS dur_sec + FROM system.compute.warehouse_events e CROSS JOIN w + WHERE e.event_time >= w.rs AND e.event_time < w.r + AND e.workspace_id IN (SELECT workspace_id FROM inv) AND e.warehouse_id IN (SELECT warehouse_id FROM inv) +), +up AS ( + SELECT workspace_id, warehouse_id, + SUM(CASE WHEN cluster_count > 0 AND dur_sec IS NOT NULL THEN dur_sec ELSE 0 END) AS wall_clock_uptime_sec, + SUM(CASE WHEN cluster_count > 0 AND dur_sec IS NOT NULL THEN cluster_count * dur_sec ELSE 0 END) AS cluster_weighted_uptime_sec + FROM ev GROUP BY workspace_id, warehouse_id +), +q AS ( + SELECT h.workspace_id, h.compute.warehouse_id AS warehouse_id, + h.execution_duration_ms, h.total_task_duration_ms, + h.waiting_at_capacity_duration_ms, h.spilled_local_bytes, + (h.execution_status = 'FINISHED' + AND (h.from_result_cache IS NULL OR h.from_result_cache = false) + AND COALESCE(h.statement_type, '') NOT IN ('SET', 'USE', 'SHOW', 'DESCRIBE', 'ALTER', 'DROP') + AND COALESCE(h.execution_duration_ms, 0) > 0) AS is_compute_query + FROM system.query.history h CROSS JOIN w + WHERE h.start_time >= w.rs AND h.start_time < w.r AND h.compute.warehouse_id IS NOT NULL + AND h.workspace_id IN (SELECT workspace_id FROM inv) AND h.compute.warehouse_id IN (SELECT warehouse_id FROM inv) +), +aq AS ( + SELECT workspace_id, warehouse_id, + COUNT(*) AS total_query_count, + SUM(COALESCE(execution_duration_ms, 0)) / 1000.0 AS active_query_sec, + COUNT_IF(is_compute_query) AS compute_query_count, + COUNT_IF(is_compute_query AND COALESCE(spilled_local_bytes, 0) = 0) AS no_spill_query_count, + COUNT_IF(is_compute_query AND COALESCE(waiting_at_capacity_duration_ms, 0) = 0) AS no_queue_query_count, + COUNT_IF(is_compute_query AND execution_duration_ms > 0 AND total_task_duration_ms > 0) AS parallelism_query_count, + MAX(CASE WHEN is_compute_query AND execution_duration_ms > 0 AND total_task_duration_ms > 0 + THEN total_task_duration_ms * 1.0 / execution_duration_ms END) AS parallelism_ratio_max, + try_element_at(sort_array(collect_list(CASE WHEN is_compute_query AND execution_duration_ms > 0 AND total_task_duration_ms > 0 + THEN total_task_duration_ms * 1.0 / execution_duration_ms END)), + CAST(FLOOR(COUNT_IF(is_compute_query AND execution_duration_ms > 0 AND total_task_duration_ms > 0) * 0.99) AS INT) + 1) + AS parallelism_ratio_p99, + try_element_at(sort_array(collect_list(CASE WHEN is_compute_query AND execution_duration_ms > 0 AND total_task_duration_ms > 0 + THEN total_task_duration_ms * 1.0 / execution_duration_ms END)), + CAST(FLOOR(COUNT_IF(is_compute_query AND execution_duration_ms > 0 AND total_task_duration_ms > 0) * 0.5) AS INT) + 1) + AS parallelism_ratio_p50 + FROM q GROUP BY workspace_id, warehouse_id +), +bill_rows AS ( + SELECT u.workspace_id, u.usage_metadata.warehouse_id AS warehouse_id, u.sku_name, u.usage_quantity, + lp.pricing.default AS price, u.usage_quantity * lp.pricing.default AS cost + FROM system.billing.usage u + CROSS JOIN w + LEFT JOIN system.billing.list_prices lp + ON u.sku_name = lp.sku_name AND u.cloud = lp.cloud + AND u.usage_start_time >= lp.price_start_time + AND (lp.price_end_time IS NULL OR u.usage_start_time < lp.price_end_time) + WHERE u.billing_origin_product = 'SQL' + AND u.usage_metadata.warehouse_id IS NOT NULL + AND u.workspace_id IN (SELECT workspace_id FROM inv) AND u.usage_metadata.warehouse_id IN (SELECT warehouse_id FROM inv) + AND CAST(u.usage_date AS TIMESTAMP) >= w.rs + AND CAST(u.usage_date AS TIMESTAMP) <= w.r + AND u.ingestion_date < w.we_date +), +bill AS ( + SELECT workspace_id, warehouse_id, SUM(usage_quantity) AS total_dbu_consumed, SUM(COALESCE(cost, 0)) AS total_cost + FROM bill_rows GROUP BY workspace_id, warehouse_id +), +type_prices AS ( + SELECT MAX(CASE WHEN sku_name LIKE '%SERVERLESS_SQL_COMPUTE%' THEN price END) AS serverless_price_per_dbu + FROM bill_rows +) +SELECT inv.warehouse_id, inv.warehouse_name, inv.warehouse_type, inv.warehouse_size, + COALESCE(up.wall_clock_uptime_sec, 0.0) AS wall_clock_uptime_sec, + COALESCE(up.cluster_weighted_uptime_sec, 0.0) AS cluster_weighted_uptime_sec, + CASE WHEN COALESCE(up.wall_clock_uptime_sec, 0) <= 0 THEN 0.0 + ELSE GREATEST(0.0, LEAST(1.0, 1.0 - COALESCE(aq.active_query_sec, 0) / up.wall_clock_uptime_sec)) END AS idle_ratio, + COALESCE(bill.total_dbu_consumed, 0.0) AS total_dbu_consumed, + COALESCE(bill.total_cost, 0.0) AS total_cost, + CASE WHEN bill.total_dbu_consumed > 0 THEN bill.total_cost / bill.total_dbu_consumed ELSE 0.0 END AS price_per_dbu, + tp.serverless_price_per_dbu, + COALESCE(aq.total_query_count, 0) AS total_query_count, + COALESCE(aq.parallelism_query_count, 0) AS parallelism_query_count, + COALESCE(aq.parallelism_ratio_p50, 0.0) AS parallelism_ratio_p50, + COALESCE(aq.parallelism_ratio_p99, 0.0) AS parallelism_ratio_p99, + COALESCE(aq.parallelism_ratio_max, 0.0) AS parallelism_ratio_max, + CASE WHEN aq.compute_query_count > 0 THEN aq.no_spill_query_count * 1.0 / aq.compute_query_count ELSE 1.0 END AS no_spill_query_pct, + CASE WHEN aq.compute_query_count > 0 THEN aq.no_queue_query_count * 1.0 / aq.compute_query_count ELSE 1.0 END AS no_queue_query_pct +FROM inv +LEFT JOIN up ON up.workspace_id = inv.workspace_id AND up.warehouse_id = inv.warehouse_id +LEFT JOIN aq ON aq.workspace_id = inv.workspace_id AND aq.warehouse_id = inv.warehouse_id +LEFT JOIN bill ON bill.workspace_id = inv.workspace_id AND bill.warehouse_id = inv.warehouse_id +CROSS JOIN type_prices tp +WHERE inv.delete_time IS NULL +ORDER BY inv.warehouse_name +` + +type RulesEngine = (input: string) => DatabricksSavingsReport + +/** The installed core's rules, or null when its binary predates them. */ +function rulesEngine(): RulesEngine | null { + const found = (core as unknown as { databricksWarehouseSavings?: unknown }).databricksWarehouseSavings + return typeof found === "function" ? (found as RulesEngine) : null +} + +/** The seven-day window the facts cover: it ends at 06:00 UTC on `end`, today's UTC date. */ +export function savingsWindow(nowMs: number): { start: string; end: string } { + const today = Math.floor(nowMs / DAY_MS) * DAY_MS + const day = (ms: number) => new Date(ms).toISOString().slice(0, 10) + return { start: day(today - LOOKBACK_DAYS * DAY_MS), end: day(today) } +} + +function num(value: unknown): number { + const parsed = typeof value === "bigint" ? Number(value) : Number(value ?? 0) + return Number.isFinite(parsed) ? parsed : 0 +} + +/** One returned row as the facts core reads; database types are coerced here, not in core. */ +export function toFacts(row: Record) { + const id = String(row.warehouse_id ?? "") + const serverless = row.serverless_price_per_dbu + return { + warehouse_id: id, + warehouse_name: String(row.warehouse_name ?? id), + warehouse_type: String(row.warehouse_type ?? ""), + warehouse_size: row.warehouse_size == null ? null : String(row.warehouse_size), + wall_clock_uptime_sec: num(row.wall_clock_uptime_sec), + cluster_weighted_uptime_sec: num(row.cluster_weighted_uptime_sec), + idle_ratio: num(row.idle_ratio), + total_dbu_consumed: num(row.total_dbu_consumed), + total_cost: num(row.total_cost), + price_per_dbu: num(row.price_per_dbu), + serverless_price_per_dbu: serverless == null ? null : num(serverless), + total_query_count: Math.round(num(row.total_query_count)), + parallelism_query_count: Math.round(num(row.parallelism_query_count)), + parallelism_ratio_p50: num(row.parallelism_ratio_p50), + parallelism_ratio_p99: num(row.parallelism_ratio_p99), + parallelism_ratio_max: num(row.parallelism_ratio_max), + no_spill_query_pct: num(row.no_spill_query_pct), + no_queue_query_pct: num(row.no_queue_query_pct), + } +} + +export async function databricksSavings(params: DatabricksSavingsParams): Promise { + const window = savingsWindow(params.now_ms ?? Date.now()) + const failed = (error: string): DatabricksSavingsResult => ({ success: false, window, error }) + + const resolved = resolveFinopsWarehouse({ + requested: params.warehouse, + supportedTypes: SUPPORTED_TYPES, + operationName: "The Databricks savings check", + }) + if (resolved.kind === "error") return failed(resolved.error) + + const config = Registry.getConfig(resolved.warehouse) as Record | undefined + const path = String(config?.http_path ?? config?.httpPath ?? "") + const runsOn = WAREHOUSE_PATH.exec(path)?.[1] + if (!runsOn) { + return failed( + `Connection ${JSON.stringify(resolved.warehouse)} does not go through a SQL warehouse. ` + + "Its http_path must name one (/sql/1.0/warehouses/): the check reads that warehouse's workspace.", + ) + } + + const rules = rulesEngine() + if (!rules) { + return failed("This altimate-code build's altimate-core has no Databricks savings rules. Update altimate-code, then try again.") + } + + try { + const connector = await Registry.get(resolved.warehouse) + const result = await connector.execute(DATABRICKS_WAREHOUSE_FACTS_SQL, 10_000, [window.end, runsOn]) + const rows = result.rows.map((row) => Object.fromEntries(result.columns.map((name, at) => [name, row[at]]))) + if (rows.length === 0) { + return failed(`No SQL warehouses were found for warehouse ${runsOn} in system.compute.warehouses.`) + } + const report = rules(JSON.stringify({ lookback_days: LOOKBACK_DAYS, warehouses: rows.map(toFacts) })) + return { success: true, window, warehouse: resolved.warehouse, ran_on: runsOn, warehouses_checked: rows.length, report } + } catch (e) { + return failed(e instanceof Error ? e.message : String(e)) + } +} diff --git a/packages/opencode/src/altimate/native/finops/register.ts b/packages/opencode/src/altimate/native/finops/register.ts index 8ee7f05d80..24aa1d71b5 100644 --- a/packages/opencode/src/altimate/native/finops/register.ts +++ b/packages/opencode/src/altimate/native/finops/register.ts @@ -6,6 +6,7 @@ import { register } from "../dispatcher" import { getQueryHistory } from "./query-history" import { analyzeCredits, getExpensiveQueries } from "./credit-analyzer" import { adviseWarehouse } from "./warehouse-advisor" +import { databricksSavings } from "./databricks-savings" import { findUnusedResources } from "./unused-resources" import { queryGrants, queryRoleHierarchy, queryUserRoles } from "./role-access" import type { @@ -17,6 +18,8 @@ import type { ExpensiveQueriesResult, WarehouseAdvisorParams, WarehouseAdvisorResult, + DatabricksSavingsParams, + DatabricksSavingsResult, UnusedResourcesParams, UnusedResourcesResult, RoleGrantsParams, @@ -46,6 +49,10 @@ register("finops.warehouse_advice", async (params: WarehouseAdvisorParams): Prom return adviseWarehouse(params) }) +register("finops.databricks_savings", async (params: DatabricksSavingsParams): Promise => { + return databricksSavings(params) +}) + register("finops.unused_resources", async (params: UnusedResourcesParams): Promise => { return findUnusedResources(params) }) diff --git a/packages/opencode/src/altimate/native/types.ts b/packages/opencode/src/altimate/native/types.ts index 09334260f7..26c5c3ebda 100644 --- a/packages/opencode/src/altimate/native/types.ts +++ b/packages/opencode/src/altimate/native/types.ts @@ -635,6 +635,49 @@ export interface ExpensiveQueriesResult { error?: string } +// --- FinOps: Databricks SQL warehouse savings --- + +export interface DatabricksSavingsParams { + /** Databricks connection name. If omitted, the first Databricks connection is used. */ + warehouse?: string + /** Clock override for tests; the window ends at 06:00 UTC on this instant's UTC date. */ + now_ms?: number +} + +/** One saving on one SQL warehouse, as altimate-core reports it. */ +export interface DatabricksSavingsFinding { + check_id: "wh_classic_pro_to_serverless" | "wh_oversized" + title: string + warehouse_id: string + warehouse_name: string + severity: "critical" | "high" | "medium" | "low" + daily_min: number + daily_max: number + monthly_min: number + monthly_max: number + evidence: Record & { kind: "serverless" | "over_sized" } +} + +export interface DatabricksSavingsReport { + lookback_days: number + /** Largest monthly saving first. */ + findings: DatabricksSavingsFinding[] + /** Each warehouse counted once, at its largest low end and largest high end. */ + total: { monthly_min: number; monthly_max: number; warehouses: number } +} + +export interface DatabricksSavingsResult { + success: boolean + /** The days the facts cover: `start` inclusive, `end` at 06:00 UTC (YYYY-MM-DD). */ + window: { start: string; end: string } + /** The connection that ran the check, and the SQL warehouse it runs on. */ + warehouse?: string + ran_on?: string + warehouses_checked?: number + report?: DatabricksSavingsReport + error?: string +} + // --- FinOps: Warehouse Advisor --- export interface WarehouseAdvisorParams { @@ -1254,6 +1297,7 @@ export const BridgeMethods = { "finops.analyze_credits": {} as { params: CreditAnalysisParams; result: CreditAnalysisResult }, "finops.expensive_queries": {} as { params: ExpensiveQueriesParams; result: ExpensiveQueriesResult }, "finops.warehouse_advice": {} as { params: WarehouseAdvisorParams; result: WarehouseAdvisorResult }, + "finops.databricks_savings": {} as { params: DatabricksSavingsParams; result: DatabricksSavingsResult }, "finops.unused_resources": {} as { params: UnusedResourcesParams; result: UnusedResourcesResult }, "finops.role_grants": {} as { params: RoleGrantsParams; result: RoleGrantsResult }, "finops.role_hierarchy": {} as { params: RoleHierarchyParams; result: RoleHierarchyResult }, diff --git a/packages/opencode/src/altimate/tool-cli/allow-list.ts b/packages/opencode/src/altimate/tool-cli/allow-list.ts index 0bb4f18473..f62a6b2d08 100644 --- a/packages/opencode/src/altimate/tool-cli/allow-list.ts +++ b/packages/opencode/src/altimate/tool-cli/allow-list.ts @@ -49,6 +49,7 @@ import { DbtSqlGraphTool } from "../tools/dbt-sql-graph" import { DbtUnitTestGenTool } from "../tools/dbt-unit-test-gen" import { DocsShareCommentsTool } from "../tools/docs-share-comments" import { FinopsAnalyzeCreditsTool } from "../tools/finops-analyze-credits" +import { FinopsDatabricksSavingsTool } from "../tools/finops-databricks-savings" import { FinopsExpensiveQueriesTool } from "../tools/finops-expensive-queries" import { FinopsQueryHistoryTool } from "../tools/finops-query-history" import { FinopsRoleGrantsTool, FinopsRoleHierarchyTool, FinopsUserRolesTool } from "../tools/finops-role-access" @@ -129,6 +130,7 @@ export const EXPOSED: Record = { dbt_unit_test_gen: DbtUnitTestGenTool, docs_share_comments: DocsShareCommentsTool, finops_analyze_credits: FinopsAnalyzeCreditsTool, + finops_databricks_savings: FinopsDatabricksSavingsTool, finops_expensive_queries: FinopsExpensiveQueriesTool, finops_query_history: FinopsQueryHistoryTool, finops_role_grants: FinopsRoleGrantsTool, diff --git a/packages/opencode/src/altimate/tools/finops-databricks-savings.ts b/packages/opencode/src/altimate/tools/finops-databricks-savings.ts new file mode 100644 index 0000000000..63d9a0b28e --- /dev/null +++ b/packages/opencode/src/altimate/tools/finops-databricks-savings.ts @@ -0,0 +1,115 @@ +// altimate_change - new file +import z from "zod" +import { Tool } from "../../tool/tool" +import { Dispatcher } from "../native" +import type { DatabricksSavingsFinding, DatabricksSavingsReport, DatabricksSavingsResult } from "../native/types" + +/** One shape for every outcome, so a reader of the metadata never guesses which fields exist. */ +type SavingsMetadata = { + success: boolean + window?: { start: string; end: string } + warehouse?: string + ran_on?: string + warehouses_checked?: number + report?: DatabricksSavingsReport + error?: string +} + +function dollars(value: number): string { + return value >= 100 + ? `$${Math.round(value).toLocaleString("en-US")}` + : `$${value.toFixed(2)}` +} + +function range(low: number, high: number): string { + return dollars(low) === dollars(high) ? dollars(high) : `${dollars(low)}–${dollars(high)}` +} + +function percent(share: unknown): string { + return `${Math.round(Number(share) * 100)}%` +} + +function size(value: unknown): string { + return String(value ?? "?").replace(/_/g, "-") +} + +/** What a finding rests on, one short line each. */ +export function evidenceLines(finding: DatabricksSavingsFinding): string[] { + const e = finding.evidence as Record & { kind: string } + if (e.kind === "serverless") { + return [ + `${e.warehouse_type} ${size(e.warehouse_size)} warehouse, ${percent(e.idle_ratio)} idle over ${Number(e.wall_clock_uptime_hours).toFixed(1)} running hours (${Number(e.total_query_count).toLocaleString("en-US")} queries)`, + `$${Number(e.current_dbu_cost).toFixed(2)} in DBUs over the window; serverless would bill $${Number(e.projected_serverless_dbu_cost).toFixed(2)} for the busy time alone`, + ] + } + return [ + `${size(e.current_size)} warehouse used at most ${(Number(e.p99_utilization) * 100).toFixed(1)}% of its cores (p99 over ${Number(e.parallelism_query_count).toLocaleString("en-US")} queries)`, + `${percent(e.no_spill_query_pct)} of queries never spilled to disk, ${percent(e.no_queue_query_pct)} never queued`, + `${size(e.recommended_size)} is enough: ${e.current_dbu_per_hour} → ${e.recommended_dbu_per_hour} DBU an hour, on $${Number(e.total_cost).toFixed(2)} spent over the window`, + ] +} + +export function formatSavings(result: DatabricksSavingsResult): string { + const report = result.report + if (!report) return "No report." + const lines: string[] = [ + `Databricks SQL warehouse savings, ${result.warehouses_checked ?? 0} warehouses checked, the ${report.lookback_days} days to ${result.window.end} 06:00 UTC, at list price.`, + ] + if (report.findings.length === 0) { + lines.push("No savings found by the Serverless and Over-Sized checks.") + return lines.join("\n") + } + lines.push( + `Total: ${range(report.total.monthly_min, report.total.monthly_max)} a month on ${report.total.warehouses} warehouse${report.total.warehouses === 1 ? "" : "s"}, each warehouse counted once at its largest saving.`, + "", + ) + for (const finding of report.findings) { + lines.push(`${finding.warehouse_name} · ${finding.title} · ${range(finding.monthly_min, finding.monthly_max)} a month (${finding.severity})`) + for (const line of evidenceLines(finding)) lines.push(` ${line}`) + } + return lines.join("\n") +} + +export const FinopsDatabricksSavingsTool = Tool.define("finops_databricks_savings", { + description: + "Check a Databricks workspace's SQL warehouses for savings with Altimate's rules: Classic/Pro warehouses that would cost less as serverless, and warehouses larger than their queries need. Read-only; returns each finding with its monthly dollar range and the measured evidence.", + parameters: z.object({ + warehouse: z + .string() + .optional() + .describe("Databricks connection name. Optional — omit to use the first configured Databricks connection."), + }), + async execute(args) { + try { + const result = await Dispatcher.call("finops.databricks_savings", { warehouse: args.warehouse }) + if (!result.success || !result.report) { + const error = result.error ?? "Unknown error" + return { + title: "Databricks Savings: FAILED", + metadata: { success: false, window: result.window, error } as SavingsMetadata, + output: `The Databricks savings check failed: ${error}`, + } + } + const total = result.report.total + return { + title: `Databricks Savings: ${range(total.monthly_min, total.monthly_max)} a month on ${total.warehouses} warehouse(s)`, + metadata: { + success: true, + window: result.window, + warehouse: result.warehouse, + ran_on: result.ran_on, + warehouses_checked: result.warehouses_checked, + report: result.report, + } as SavingsMetadata, + output: formatSavings(result), + } + } catch (e) { + const msg = e instanceof Error ? e.message : String(e) + return { + title: "Databricks Savings: ERROR", + metadata: { success: false, error: msg } as SavingsMetadata, + output: `The Databricks savings check failed: ${msg}`, + } + } + }, +}) diff --git a/packages/opencode/src/tool/registry.ts b/packages/opencode/src/tool/registry.ts index 06af06b5c9..db2e3b238e 100644 --- a/packages/opencode/src/tool/registry.ts +++ b/packages/opencode/src/tool/registry.ts @@ -106,6 +106,7 @@ import { FinopsQueryHistoryTool } from "../altimate/tools/finops-query-history" import { FinopsAnalyzeCreditsTool } from "../altimate/tools/finops-analyze-credits" import { FinopsExpensiveQueriesTool } from "../altimate/tools/finops-expensive-queries" import { FinopsWarehouseAdviceTool } from "../altimate/tools/finops-warehouse-advice" +import { FinopsDatabricksSavingsTool } from "../altimate/tools/finops-databricks-savings" import { FinopsUnusedResourcesTool } from "../altimate/tools/finops-unused-resources" import { FinopsRoleGrantsTool, @@ -490,6 +491,7 @@ export namespace ToolRegistry { FinopsAnalyzeCreditsTool, FinopsExpensiveQueriesTool, FinopsWarehouseAdviceTool, + FinopsDatabricksSavingsTool, FinopsUnusedResourcesTool, FinopsRoleGrantsTool, FinopsRoleHierarchyTool, diff --git a/packages/opencode/test/agent/analyst-data-explorer.test.ts b/packages/opencode/test/agent/analyst-data-explorer.test.ts index bacd21a74f..8943948662 100644 --- a/packages/opencode/test/agent/analyst-data-explorer.test.ts +++ b/packages/opencode/test/agent/analyst-data-explorer.test.ts @@ -419,6 +419,7 @@ const analystConfigEnabled = new Set([ "dbt_profiles", "dbt_unit_test_gen", "finops_analyze_credits", + "finops_databricks_savings", "finops_expensive_queries", "finops_query_history", "finops_role_grants", diff --git a/packages/opencode/test/altimate/finops-databricks-e2e.test.ts b/packages/opencode/test/altimate/finops-databricks-e2e.test.ts index ec9a19dbc0..fba4e58f89 100644 --- a/packages/opencode/test/altimate/finops-databricks-e2e.test.ts +++ b/packages/opencode/test/altimate/finops-databricks-e2e.test.ts @@ -23,6 +23,7 @@ import { analyzeCredits, getExpensiveQueries } from "../../src/altimate/native/f import { adviseWarehouse } from "../../src/altimate/native/finops/warehouse-advisor" import { findUnusedResources } from "../../src/altimate/native/finops/unused-resources" import { queryGrants } from "../../src/altimate/native/finops/role-access" +import { databricksSavings } from "../../src/altimate/native/finops/databricks-savings" const DB_CONFIG = process.env.ALTIMATE_CODE_CONN_DATABRICKS_TEST const HAS_DATABRICKS = !!DB_CONFIG @@ -204,4 +205,18 @@ describe.skipIf(!HAS_DATABRICKS)("Databricks FinOps Integration", () => { } }) }) + // ------------------------------------------------------------------------- + // Databricks SQL warehouse savings (facts measured here, rules in altimate-core) + // ------------------------------------------------------------------------- + describe("databricks_savings", () => { + test("measures every warehouse of the workspace and returns core's report", async () => { + const result = await databricksSavings({ warehouse: WH }) + expectValidResult(result) + if (result.success) { + expect(result.warehouses_checked).toBeGreaterThan(0) + expect(Array.isArray(result.report!.findings)).toBe(true) + expect(result.report!.total.warehouses).toBe(new Set(result.report!.findings.map((f) => f.warehouse_id)).size) + } + }, 300000) + }) }) diff --git a/packages/opencode/test/altimate/finops-databricks-savings.test.ts b/packages/opencode/test/altimate/finops-databricks-savings.test.ts new file mode 100644 index 0000000000..8865901e20 --- /dev/null +++ b/packages/opencode/test/altimate/finops-databricks-savings.test.ts @@ -0,0 +1,178 @@ +import { afterEach, beforeEach, describe, expect, test } from "bun:test" +import { initTool } from "./tool-fixture" +import * as Dispatcher from "../../src/altimate/native/dispatcher" +import * as Registry from "../../src/altimate/native/connections/registry" +import { + DATABRICKS_WAREHOUSE_FACTS_SQL, + databricksSavings, + savingsWindow, + toFacts, +} from "../../src/altimate/native/finops/databricks-savings" +import { FinopsDatabricksSavingsTool } from "../../src/altimate/tools/finops-databricks-savings" +import type { DatabricksSavingsResult } from "../../src/altimate/native/types" + +function stubCtx(): any { + return { sessionID: "test", messageID: "test", agent: "test", abort: new AbortController().signal, metadata: () => {}, ask: async () => {} } +} + +// What altimate-core returned for warehouse `Classic` on dbxmega for the seven days to +// 2026-11-12 06:00 UTC (findings rounded to the cent for the fixture). +const REPORT: DatabricksSavingsResult = { + success: true, + window: { start: "2026-11-05", end: "2026-11-12" }, + warehouse: "dbxmega", + ran_on: "4ec45aec4ab985ab", + warehouses_checked: 5, + report: { + lookback_days: 7, + findings: [ + { + check_id: "wh_classic_pro_to_serverless", + title: "Migrate Classic/Pro to Serverless", + warehouse_id: "a3688f261647f580", + warehouse_name: "Classic", + severity: "critical", + daily_min: 1.773465, + daily_max: 2.216832, + monthly_min: 53.20, + monthly_max: 66.50, + evidence: { + kind: "serverless", + warehouse_type: "CLASSIC", + warehouse_size: "SMALL", + idle_ratio: 0.8926, + wall_clock_uptime_hours: 10.897, + total_query_count: 3580, + price_per_dbu: 0.22, + serverless_price_per_dbu: 0.7, + current_dbu_cost: 23.58, + projected_serverless_dbu_cost: 8.06, + wasted_cost: 15.52, + }, + }, + { + check_id: "wh_oversized", + title: "Over-Sized Warehouse", + warehouse_id: "a3688f261647f580", + warehouse_name: "Classic", + severity: "high", + daily_min: 0.898177, + daily_max: 1.57181, + monthly_min: 26.95, + monthly_max: 47.15, + evidence: { + kind: "over_sized", + current_size: "SMALL", + recommended_size: "2X_SMALL", + available_cores: 32, + p99_utilization: 0.0834, + parallelism_query_count: 3208, + no_spill_query_pct: 1, + no_queue_query_pct: 1, + current_dbu_per_hour: 12, + recommended_dbu_per_hour: 4, + total_cost: 23.58, + wasted_cost: 15.72, + }, + }, + ], + total: { monthly_min: 53.2, monthly_max: 66.5, warehouses: 1 }, + }, +} + +describe("finops_databricks_savings tool", () => { + beforeEach(() => Dispatcher.reset()) + + test("gives Claude the total, each finding with its evidence, and the whole report as metadata", async () => { + Dispatcher.register("finops.databricks_savings" as any, async () => REPORT) + const tool = await initTool(FinopsDatabricksSavingsTool) + const result = await tool.execute({}, stubCtx()) + + expect(result.title).toBe("Databricks Savings: $53.20–$66.50 a month on 1 warehouse(s)") + expect(result.output).toContain("Total: $53.20–$66.50 a month on 1 warehouse, each warehouse counted once") + expect(result.output).toContain("Classic · Migrate Classic/Pro to Serverless · $53.20–$66.50 a month (critical)") + expect(result.output).toContain("CLASSIC SMALL warehouse, 89% idle over 10.9 running hours (3,580 queries)") + expect(result.output).toContain("2X-SMALL is enough: 12 → 4 DBU an hour") + expect(result.metadata.report).toEqual(REPORT.report) + expect(result.metadata.window).toEqual({ start: "2026-11-05", end: "2026-11-12" }) + }) + + test("no findings says so and still reports how many warehouses were checked", async () => { + Dispatcher.register("finops.databricks_savings" as any, async () => ({ + ...REPORT, + report: { lookback_days: 7, findings: [], total: { monthly_min: 0, monthly_max: 0, warehouses: 0 } }, + })) + const tool = await initTool(FinopsDatabricksSavingsTool) + const result = await tool.execute({}, stubCtx()) + expect(result.output).toContain("5 warehouses checked") + expect(result.output).toContain("No savings found") + }) + + test("a failure is reported with its reason, not as no savings", async () => { + Dispatcher.register("finops.databricks_savings" as any, async () => ({ + success: false, + window: { start: "2026-11-05", end: "2026-11-12" }, + error: "[TABLE_OR_VIEW_NOT_FOUND] The table or view `system`.`compute`.`warehouse_events` cannot be found.", + })) + const tool = await initTool(FinopsDatabricksSavingsTool) + const result = await tool.execute({}, stubCtx()) + expect(result.title).toBe("Databricks Savings: FAILED") + expect(result.metadata.error).toContain("TABLE_OR_VIEW_NOT_FOUND") + expect(result.output).not.toContain("No savings found") + }) +}) + +describe("databricks savings handler", () => { + afterEach(() => Registry.reset()) + + test("the window is the seven days to 06:00 UTC today", () => { + expect(savingsWindow(Date.parse("2026-11-12T10:00:00Z"))).toEqual({ start: "2026-11-05", end: "2026-11-12" }) + }) + + test("the facts query is one read-only statement taking the window end and the warehouse id, in that order", () => { + expect(DATABRICKS_WAREHOUSE_FACTS_SQL.trim()).toMatch(/^WITH\b/) + expect(DATABRICKS_WAREHOUSE_FACTS_SQL).not.toMatch(/;\s*\S/) + const markers = [...DATABRICKS_WAREHOUSE_FACTS_SQL.matchAll(/\?/g)].map((m) => m.index!) + expect(markers.length).toBe(2) + expect(DATABRICKS_WAREHOUSE_FACTS_SQL.slice(0, markers[0]!)).toContain("CAST(") + expect(DATABRICKS_WAREHOUSE_FACTS_SQL.slice(markers[0]!, markers[1]!)).toContain("WHERE workspace_id = (SELECT MAX(workspace_id)") + }) + + test("database values are coerced to the numbers core reads; no serverless price stays absent", () => { + const facts = toFacts({ + warehouse_id: "a1", + warehouse_name: null, + warehouse_type: "PRO", + warehouse_size: "SMALL", + wall_clock_uptime_sec: "39229.0", + total_query_count: 3580n, + parallelism_query_count: "3208", + idle_ratio: 0.89, + serverless_price_per_dbu: null, + no_spill_query_pct: "1.0000000000000000", + }) + expect(facts.warehouse_name).toBe("a1") + expect(facts.wall_clock_uptime_sec).toBe(39229) + expect(facts.total_query_count).toBe(3580) + expect(facts.parallelism_query_count).toBe(3208) + expect(facts.serverless_price_per_dbu).toBeNull() + expect(facts.no_spill_query_pct).toBe(1) + expect(facts.total_cost).toBe(0) + }) + + test("with no Databricks connection it says how to add one", async () => { + Registry.setConfigs({}) + const result = await databricksSavings({}) + expect(result.success).toBe(false) + expect(result.error).toContain("warehouse_add") + }) + + test("a connection that does not go through a SQL warehouse is refused before any query", async () => { + Registry.setConfigs({ + dbx: { type: "databricks", server_hostname: "dbc-0.cloud.databricks.com", http_path: "/sql/protocolv1/o/123/0101-abc", access_token: "x" }, + }) + const result = await databricksSavings({ warehouse: "dbx" }) + expect(result.success).toBe(false) + expect(result.error).toContain("/sql/1.0/warehouses/") + }) +}) From 3cbff41abba527ecd2f35904ea97d3cd980d989c Mon Sep 17 00:00:00 2001 From: Shreyas Telkar Date: Tue, 17 Nov 2026 16:05:44 -0800 Subject: [PATCH 02/34] feat: `finops_databricks_savings`: idle clusters, jobs on all-purpose compute, what-if - measures per all-purpose cluster (node timeline idle minutes, CPU, auto-termination, all-purpose cost outside jobs) and per job (cost by product, share of its main cluster, runs, trigger, the workspace's jobs rate), seven full days, cut to the connection warehouse's workspace - altimate-core's `databricksSavings` judges warehouses, clusters and jobs in one report; the output names each finding's resource and kind, lists what each flagged warehouse would cost at every smaller size and as serverless, and names any check whose system tables could not be read - metadata carries each warehouse's type, size and window cost so a later run can tell what changed - with no connection named, a Databricks connection that runs through a SQL warehouse is preferred; the description says the check may start it - docs: finops-tools.md entry Co-Authored-By: Claude Opus 5.5 --- .../data-engineering/tools/finops-tools.md | 24 ++ .../native/finops/databricks-savings.ts | 240 +++++++++++++++++- .../opencode/src/altimate/native/types.ts | 31 ++- .../tools/finops-databricks-savings.ts | 75 ++++-- .../altimate/finops-databricks-e2e.test.ts | 2 +- .../finops-databricks-savings.test.ts | 101 +++++++- 6 files changed, 429 insertions(+), 44 deletions(-) diff --git a/docs/docs/data-engineering/tools/finops-tools.md b/docs/docs/data-engineering/tools/finops-tools.md index 2824e46231..54b9bc520f 100644 --- a/docs/docs/data-engineering/tools/finops-tools.md +++ b/docs/docs/data-engineering/tools/finops-tools.md @@ -178,6 +178,30 @@ Recommendations: --- +## finops_databricks_savings + +Check a Databricks workspace for savings with Altimate's rules, on a Databricks connection whose `http_path` is a SQL warehouse (`/sql/1.0/warehouses/`). It reads seven days of system tables (`system.billing`, `system.compute`, `system.query`, `system.lakeflow`) for the workspace that warehouse belongs to, on that warehouse, which Databricks starts if it is stopped. Nothing is written. Four checks: + +- **Migrate Classic/Pro to Serverless:** a Classic or Pro warehouse that bills for time it sits idle. +- **Over-Sized Warehouse:** a warehouse whose queries never use the cores it pays for. +- **Idle Cluster Detection:** an all-purpose cluster with idle workers and auto-termination off or an hour or more. +- **Job Running on All-Purpose Compute:** a scheduled job that would bill at the jobs-compute rate on a job cluster. + +The tool measures each resource's usage; altimate-core decides what qualifies and what it saves. Prices are list prices. Each resource is counted once in the total, and each flagged warehouse also gets what it would cost at every smaller size and as serverless. A check whose tables the connection cannot read is named as not checked; the others still report. + +``` +> finops_databricks_savings --warehouse prod_dbx + +Databricks savings: 5 SQL warehouses, 3 all-purpose clusters, 12 jobs checked, the 7 days to 2026-11-17, at list price. +Total: $85.17–$106 a month on 1 resource, each counted once at its largest saving. + +Classic (sql warehouse) · Migrate Classic/Pro to Serverless · $85.17–$106 a month (critical) + CLASSIC SMALL warehouse, 95% idle over 10.8 running hours (2,988 queries) + $29.29 in DBUs over the window; serverless would bill $4.44 for the busy time alone +``` + +--- + ## finops_role_grants Analyze role permissions and access patterns (RBAC). diff --git a/packages/opencode/src/altimate/native/finops/databricks-savings.ts b/packages/opencode/src/altimate/native/finops/databricks-savings.ts index e7e1a13774..02954bd67e 100644 --- a/packages/opencode/src/altimate/native/finops/databricks-savings.ts +++ b/packages/opencode/src/altimate/native/finops/databricks-savings.ts @@ -139,11 +139,152 @@ WHERE inv.delete_time IS NULL ORDER BY inv.warehouse_name ` +/** The workspace of the warehouse the connection runs on: every input is cut to it. */ +const WORKSPACE_OF_WAREHOUSE = "(SELECT MAX(workspace_id) FROM system.compute.warehouses WHERE warehouse_id = ?)" + +/** List price of a usage row: the price in force when the usage started. */ +const LIST_PRICE_JOIN = `LEFT JOIN system.billing.list_prices lp + ON u.sku_name = lp.sku_name AND u.cloud = lp.cloud + AND u.usage_start_time >= lp.price_start_time + AND (lp.price_end_time IS NULL OR u.usage_start_time < lp.price_end_time)` + +/** + * Per all-purpose cluster (created from the UI or API, not deleted) of the workspace: worker node + * minutes, active days, idle worker minutes (CPU under 5%, CPU wait under 5%, under 10 MB of network + * a minute), CPU averages, and the all-purpose cost billed outside job runs. Seven full days before + * the window end. Parameters: the window end, then the warehouse id. + */ +export const DATABRICKS_CLUSTER_FACTS_SQL = ` +WITH p AS ( + SELECT CAST(? AS DATE) AS we_date +), +ws AS ( + SELECT ${WORKSPACE_OF_WAREHOUSE} AS workspace_id +), +inv AS ( + SELECT cluster_id, cluster_name, auto_termination_minutes, cluster_source, delete_time + FROM system.compute.clusters + WHERE workspace_id = (SELECT workspace_id FROM ws) + QUALIFY ROW_NUMBER() OVER (PARTITION BY cluster_id ORDER BY change_time DESC) = 1 +), +nodes AS ( + SELECT n.cluster_id, n.start_time, n.end_time, + COALESCE(n.cpu_user_percent, 0) + COALESCE(n.cpu_system_percent, 0) AS cpu_active, + COALESCE(n.cpu_wait_percent, 0) AS cpu_wait, + COALESCE(n.network_sent_bytes, 0) + COALESCE(n.network_received_bytes, 0) AS network_bytes + FROM system.compute.node_timeline n CROSS JOIN p + WHERE n.workspace_id = (SELECT workspace_id FROM ws) + AND n.start_time >= CAST(DATE_SUB(p.we_date, 7) AS TIMESTAMP) AND n.start_time < CAST(p.we_date AS TIMESTAMP) + AND NOT COALESCE(n.driver, false) + AND n.cluster_id IN (SELECT cluster_id FROM inv) +), +util AS ( + SELECT cluster_id, + COUNT(DISTINCT CAST(start_time AS DATE)) AS active_util_days, + SUM(timestampdiff(MINUTE, start_time, end_time)) AS total_node_minutes, + COUNT(*) AS total_worker_minutes, + COUNT_IF(cpu_active < 5.0 AND cpu_wait < 5.0 AND network_bytes < 10485760) AS idle_node_minutes, + AVG(cpu_active) AS avg_cpu_active_pct, + MAX(cpu_active) AS peak_cpu_active_pct + FROM nodes GROUP BY cluster_id +), +bill AS ( + SELECT u.usage_metadata.cluster_id AS cluster_id, SUM(u.usage_quantity * lp.pricing.default) AS total_cost + FROM system.billing.usage u CROSS JOIN p + ${LIST_PRICE_JOIN} + WHERE u.workspace_id = (SELECT workspace_id FROM ws) + AND u.billing_origin_product = 'ALL_PURPOSE' + AND u.usage_metadata.cluster_id IS NOT NULL AND u.usage_metadata.job_id IS NULL + AND u.usage_date >= DATE_SUB(p.we_date, 7) AND u.usage_date < p.we_date + GROUP BY u.usage_metadata.cluster_id +) +SELECT inv.cluster_id, inv.cluster_name, inv.auto_termination_minutes, + util.total_node_minutes, util.active_util_days, util.total_worker_minutes, util.idle_node_minutes, + util.avg_cpu_active_pct, util.peak_cpu_active_pct, + COALESCE(bill.total_cost, 0.0) AS total_cost +FROM inv +JOIN util ON util.cluster_id = inv.cluster_id +LEFT JOIN bill ON bill.cluster_id = inv.cluster_id +WHERE inv.delete_time IS NULL AND inv.cluster_source IN ('UI', 'API') +ORDER BY inv.cluster_name +` + +/** + * Per job of the workspace that billed in the window: its cost on every product and on all-purpose + * compute, its share of the all-purpose cluster it used most, its runs and trigger, and the + * workspace's jobs-compute rate. Seven full days before the window end. Parameters: the window end, + * then the warehouse id. + */ +export const DATABRICKS_JOB_FACTS_SQL = ` +WITH p AS ( + SELECT CAST(? AS DATE) AS we_date +), +ws AS ( + SELECT ${WORKSPACE_OF_WAREHOUSE} AS workspace_id +), +bill AS ( + SELECT u.usage_metadata.job_id AS job_id, u.usage_metadata.cluster_id AS cluster_id, u.billing_origin_product AS product, + u.usage_quantity AS dbus, u.usage_quantity * lp.pricing.default AS cost + FROM system.billing.usage u CROSS JOIN p + ${LIST_PRICE_JOIN} + WHERE u.workspace_id = (SELECT workspace_id FROM ws) + AND u.usage_date >= DATE_SUB(p.we_date, 7) AND u.usage_date < p.we_date +), +job_bill AS ( + SELECT job_id, + SUM(COALESCE(cost, 0)) AS job_total_cost, + SUM(CASE WHEN product = 'ALL_PURPOSE' THEN COALESCE(cost, 0) ELSE 0 END) AS job_all_purpose_cost, + SUM(CASE WHEN product = 'ALL_PURPOSE' THEN dbus ELSE 0 END) AS job_all_purpose_dbus + FROM bill WHERE job_id IS NOT NULL AND job_id <> '' GROUP BY job_id +), +job_cluster AS ( + SELECT job_id, cluster_id, SUM(COALESCE(cost, 0)) AS cost + FROM bill WHERE product = 'ALL_PURPOSE' AND job_id IS NOT NULL AND job_id <> '' AND cluster_id IS NOT NULL + GROUP BY job_id, cluster_id +), +main_cluster AS ( + SELECT job_id, cluster_id, cost FROM job_cluster + QUALIFY ROW_NUMBER() OVER (PARTITION BY job_id ORDER BY cost DESC, cluster_id) = 1 +), +cluster_total AS ( + SELECT cluster_id, SUM(COALESCE(cost, 0)) AS cost + FROM bill WHERE product = 'ALL_PURPOSE' AND cluster_id IS NOT NULL GROUP BY cluster_id +), +jobs_rate AS ( + SELECT CASE WHEN SUM(dbus) > 0 THEN SUM(COALESCE(cost, 0)) / SUM(dbus) END AS rate FROM bill WHERE product = 'JOBS' +), +runs AS ( + SELECT r.job_id, COUNT(DISTINCT r.run_id) AS run_count, MAX(r.trigger_type) AS run_trigger + FROM system.lakeflow.job_run_timeline r CROSS JOIN p + WHERE r.workspace_id = (SELECT workspace_id FROM ws) + AND r.period_start_time >= CAST(DATE_SUB(p.we_date, 7) AS TIMESTAMP) AND r.period_start_time < CAST(p.we_date AS TIMESTAMP) + GROUP BY r.job_id +), +inv AS ( + SELECT job_id, name, trigger_type FROM system.lakeflow.jobs + WHERE workspace_id = (SELECT workspace_id FROM ws) + QUALIFY ROW_NUMBER() OVER (PARTITION BY job_id ORDER BY change_time DESC) = 1 +) +SELECT jb.job_id, COALESCE(inv.name, 'job ' || jb.job_id) AS job_name, + COALESCE(inv.trigger_type, runs.run_trigger) AS trigger_type, + COALESCE(runs.run_count, 0) AS run_count, + jb.job_total_cost, jb.job_all_purpose_cost, jb.job_all_purpose_dbus, + CASE WHEN mc.cluster_id IS NULL OR COALESCE(ct.cost, 0) <= 0 THEN 1.0 ELSE mc.cost / ct.cost END AS job_share_of_cluster, + (SELECT rate FROM jobs_rate) AS jobs_rate_per_dbu +FROM job_bill jb +LEFT JOIN runs ON runs.job_id = jb.job_id +LEFT JOIN inv ON inv.job_id = jb.job_id +LEFT JOIN main_cluster mc ON mc.job_id = jb.job_id +LEFT JOIN cluster_total ct ON ct.cluster_id = mc.cluster_id +WHERE jb.job_all_purpose_cost > 0 +ORDER BY jb.job_all_purpose_cost DESC +` + type RulesEngine = (input: string) => DatabricksSavingsReport /** The installed core's rules, or null when its binary predates them. */ function rulesEngine(): RulesEngine | null { - const found = (core as unknown as { databricksWarehouseSavings?: unknown }).databricksWarehouseSavings + const found = (core as unknown as { databricksSavings?: unknown }).databricksSavings return typeof found === "function" ? (found as RulesEngine) : null } @@ -185,12 +326,60 @@ export function toFacts(row: Record) { } } +/** One cluster row as the facts core reads. */ +export function toClusterFacts(row: Record) { + const id = String(row.cluster_id ?? "") + const autoTermination = row.auto_termination_minutes + return { + cluster_id: id, + cluster_name: String(row.cluster_name ?? id), + auto_termination_minutes: autoTermination == null ? null : Math.max(0, Math.round(num(autoTermination))), + total_node_minutes: num(row.total_node_minutes), + active_util_days: Math.round(num(row.active_util_days)), + total_worker_minutes: num(row.total_worker_minutes), + idle_node_minutes: num(row.idle_node_minutes), + avg_cpu_active_pct: num(row.avg_cpu_active_pct), + peak_cpu_active_pct: num(row.peak_cpu_active_pct), + total_cost: num(row.total_cost), + } +} + +/** One job row as the facts core reads. */ +export function toJobFacts(row: Record) { + const id = String(row.job_id ?? "") + const rate = row.jobs_rate_per_dbu + return { + job_id: id, + job_name: String(row.job_name ?? `job ${id}`), + trigger_type: row.trigger_type == null ? null : String(row.trigger_type), + run_count: Math.round(num(row.run_count)), + job_total_cost: num(row.job_total_cost), + job_all_purpose_cost: num(row.job_all_purpose_cost), + job_all_purpose_dbus: num(row.job_all_purpose_dbus), + job_share_of_cluster: Math.min(1, num(row.job_share_of_cluster)), + jobs_rate_per_dbu: rate == null ? null : num(rate), + } +} + +function records(result: { columns: string[]; rows: unknown[][] }): Record[] { + return result.rows.map((row) => Object.fromEntries(result.columns.map((name, at) => [name, row[at]]))) +} + +/** With no connection named: the first Databricks connection that runs through a SQL warehouse, if any. */ +function preferredConnection(): string | undefined { + return Registry.list().warehouses.find((one) => { + if (one.type.toLowerCase() !== "databricks") return false + const config = Registry.getConfig(one.name) as Record | undefined + return WAREHOUSE_PATH.test(String(config?.http_path ?? config?.httpPath ?? "")) + })?.name +} + export async function databricksSavings(params: DatabricksSavingsParams): Promise { const window = savingsWindow(params.now_ms ?? Date.now()) const failed = (error: string): DatabricksSavingsResult => ({ success: false, window, error }) const resolved = resolveFinopsWarehouse({ - requested: params.warehouse, + requested: params.warehouse ?? preferredConnection(), supportedTypes: SUPPORTED_TYPES, operationName: "The Databricks savings check", }) @@ -213,13 +402,50 @@ export async function databricksSavings(params: DatabricksSavingsParams): Promis try { const connector = await Registry.get(resolved.warehouse) - const result = await connector.execute(DATABRICKS_WAREHOUSE_FACTS_SQL, 10_000, [window.end, runsOn]) - const rows = result.rows.map((row) => Object.fromEntries(result.columns.map((name, at) => [name, row[at]]))) - if (rows.length === 0) { + const binds = [window.end, runsOn] + const warehouses = records(await connector.execute(DATABRICKS_WAREHOUSE_FACTS_SQL, 10_000, binds)) + if (warehouses.length === 0) { return failed(`No SQL warehouses were found for warehouse ${runsOn} in system.compute.warehouses.`) } - const report = rules(JSON.stringify({ lookback_days: LOOKBACK_DAYS, warehouses: rows.map(toFacts) })) - return { success: true, window, warehouse: resolved.warehouse, ran_on: runsOn, warehouses_checked: rows.length, report } + // A table the connection may not read stops only its own check; the rest still report. + const skipped: { check: string; reason: string }[] = [] + const optional = async (check: string, sql: string) => { + try { + return records(await connector.execute(sql, 10_000, binds)) + } catch (e) { + skipped.push({ check, reason: (e instanceof Error ? e.message : String(e)).split("\n")[0]!.slice(0, 300) }) + return [] + } + } + const clusters = await optional("Idle Cluster Detection", DATABRICKS_CLUSTER_FACTS_SQL) + const jobs = await optional("Job Running on All-Purpose Compute", DATABRICKS_JOB_FACTS_SQL) + const report = rules( + JSON.stringify({ + lookback_days: LOOKBACK_DAYS, + warehouses: warehouses.map(toFacts), + clusters: clusters.map(toClusterFacts), + jobs: jobs.map(toJobFacts), + }), + ) + return { + success: true, + window, + warehouse: resolved.warehouse, + ran_on: runsOn, + warehouses_checked: warehouses.length, + clusters_checked: clusters.length, + jobs_checked: jobs.length, + skipped, + // What each warehouse is and costs now, so a later run can tell what changed. + warehouses: warehouses.map((row) => ({ + id: String(row.warehouse_id ?? ""), + name: String(row.warehouse_name ?? row.warehouse_id ?? ""), + type: String(row.warehouse_type ?? ""), + size: row.warehouse_size == null ? null : String(row.warehouse_size), + window_cost: num(row.total_cost), + })), + report, + } } catch (e) { return failed(e instanceof Error ? e.message : String(e)) } diff --git a/packages/opencode/src/altimate/native/types.ts b/packages/opencode/src/altimate/native/types.ts index 26c5c3ebda..2643b79008 100644 --- a/packages/opencode/src/altimate/native/types.ts +++ b/packages/opencode/src/altimate/native/types.ts @@ -644,26 +644,37 @@ export interface DatabricksSavingsParams { now_ms?: number } -/** One saving on one SQL warehouse, as altimate-core reports it. */ +/** One saving on one resource, as altimate-core reports it. */ export interface DatabricksSavingsFinding { - check_id: "wh_classic_pro_to_serverless" | "wh_oversized" + check_id: "wh_classic_pro_to_serverless" | "wh_oversized" | "ap_idle_cluster" | "jobs_on_all_purpose_compute" title: string - warehouse_id: string - warehouse_name: string + resource_type: "sql_warehouse" | "cluster" | "job" + resource_id: string + resource_name: string severity: "critical" | "high" | "medium" | "low" daily_min: number daily_max: number monthly_min: number monthly_max: number - evidence: Record & { kind: "serverless" | "over_sized" } + evidence: Record & { kind: "serverless" | "over_sized" | "idle_cluster" | "job_on_all_purpose" } +} + +/** A flagged SQL warehouse's cost at each alternative, as altimate-core computes it. */ +export interface DatabricksWhatIf { + resource_id: string + resource_name: string + current_label: string + current_monthly_cost: number + choices: { label: string; monthly_cost: number; monthly_saving: number; fits: boolean; cores: number; cores_needed: number }[] } export interface DatabricksSavingsReport { lookback_days: number /** Largest monthly saving first. */ findings: DatabricksSavingsFinding[] - /** Each warehouse counted once, at its largest low end and largest high end. */ - total: { monthly_min: number; monthly_max: number; warehouses: number } + /** Each resource counted once, at its largest low end and largest high end. */ + total: { monthly_min: number; monthly_max: number; resources: number } + what_if: DatabricksWhatIf[] } export interface DatabricksSavingsResult { @@ -674,6 +685,12 @@ export interface DatabricksSavingsResult { warehouse?: string ran_on?: string warehouses_checked?: number + clusters_checked?: number + jobs_checked?: number + /** Checks whose system tables the connection could not read. */ + skipped?: { check: string; reason: string }[] + /** Each SQL warehouse as it is now, and its cost over the window. */ + warehouses?: { id: string; name: string; type: string; size: string | null; window_cost: number }[] report?: DatabricksSavingsReport error?: string } diff --git a/packages/opencode/src/altimate/tools/finops-databricks-savings.ts b/packages/opencode/src/altimate/tools/finops-databricks-savings.ts index 63d9a0b28e..ec3185660f 100644 --- a/packages/opencode/src/altimate/tools/finops-databricks-savings.ts +++ b/packages/opencode/src/altimate/tools/finops-databricks-savings.ts @@ -11,6 +11,10 @@ type SavingsMetadata = { warehouse?: string ran_on?: string warehouses_checked?: number + clusters_checked?: number + jobs_checked?: number + skipped?: { check: string; reason: string }[] + warehouses?: { id: string; name: string; type: string; size: string | null; window_cost: number }[] report?: DatabricksSavingsReport error?: string } @@ -33,46 +37,75 @@ function size(value: unknown): string { return String(value ?? "?").replace(/_/g, "-") } +const percentOf = (value: unknown) => `${Math.round(Number(value))}%` + /** What a finding rests on, one short line each. */ export function evidenceLines(finding: DatabricksSavingsFinding): string[] { - const e = finding.evidence as Record & { kind: string } - if (e.kind === "serverless") { - return [ - `${e.warehouse_type} ${size(e.warehouse_size)} warehouse, ${percent(e.idle_ratio)} idle over ${Number(e.wall_clock_uptime_hours).toFixed(1)} running hours (${Number(e.total_query_count).toLocaleString("en-US")} queries)`, - `$${Number(e.current_dbu_cost).toFixed(2)} in DBUs over the window; serverless would bill $${Number(e.projected_serverless_dbu_cost).toFixed(2)} for the busy time alone`, - ] + const e = finding.evidence as Record & { kind: string } + switch (e.kind) { + case "serverless": + return [ + `${e.warehouse_type} ${size(e.warehouse_size)} warehouse, ${percent(e.idle_ratio)} idle over ${Number(e.wall_clock_uptime_hours).toFixed(1)} running hours (${Number(e.total_query_count).toLocaleString("en-US")} queries)`, + `$${Number(e.current_dbu_cost).toFixed(2)} in DBUs over the window; serverless would bill $${Number(e.projected_serverless_dbu_cost).toFixed(2)} for the busy time alone`, + ] + case "over_sized": + return [ + `${size(e.current_size)} warehouse used at most ${(Number(e.p99_utilization) * 100).toFixed(1)}% of its cores (p99 over ${Number(e.parallelism_query_count).toLocaleString("en-US")} queries)`, + `${percent(e.no_spill_query_pct)} of queries never spilled to disk, ${percent(e.no_queue_query_pct)} never queued`, + `${size(e.recommended_size)} is enough: ${e.current_dbu_per_hour} → ${e.recommended_dbu_per_hour} DBU an hour, on $${Number(e.total_cost).toFixed(2)} spent over the window`, + ] + case "idle_cluster": + return [ + `All-purpose cluster ${percent(e.idle_ratio)} idle across ${e.active_days} active days (${String(e.idle_pattern).replace(/_/g, " ")}), average worker CPU ${percentOf(e.avg_cpu_active_pct)}, peak ${percentOf(e.peak_cpu_active_pct)}`, + `Auto-termination ${e.auto_termination_minutes ? `${e.auto_termination_minutes} minutes` : "off"}; $${Number(e.total_cost).toFixed(2)} billed outside job runs over the window`, + ] + case "job_on_all_purpose": + return [ + `${e.run_count} runs (${String(e.trigger_type ?? "manual").toLowerCase()}), ${percent(e.all_purpose_share)} of its cost on all-purpose compute, ${percent(e.job_share_of_cluster)} of its cluster's`, + `$${Number(e.all_purpose_rate_per_dbu).toFixed(3)} a DBU now; a job cluster would be about $${Number(e.jobs_rate_per_dbu).toFixed(3)}${e.jobs_rate_estimated ? " (estimated: the workspace billed no jobs compute)" : ""}`, + ] + default: + return [] } - return [ - `${size(e.current_size)} warehouse used at most ${(Number(e.p99_utilization) * 100).toFixed(1)}% of its cores (p99 over ${Number(e.parallelism_query_count).toLocaleString("en-US")} queries)`, - `${percent(e.no_spill_query_pct)} of queries never spilled to disk, ${percent(e.no_queue_query_pct)} never queued`, - `${size(e.recommended_size)} is enough: ${e.current_dbu_per_hour} → ${e.recommended_dbu_per_hour} DBU an hour, on $${Number(e.total_cost).toFixed(2)} spent over the window`, - ] } export function formatSavings(result: DatabricksSavingsResult): string { const report = result.report if (!report) return "No report." - const lines: string[] = [ - `Databricks SQL warehouse savings, ${result.warehouses_checked ?? 0} warehouses checked, the ${report.lookback_days} days to ${result.window.end} 06:00 UTC, at list price.`, - ] + const counted = [ + `${result.warehouses_checked ?? 0} SQL warehouses`, + `${result.clusters_checked ?? 0} all-purpose clusters`, + `${result.jobs_checked ?? 0} jobs`, + ].join(", ") + const lines: string[] = [`Databricks savings: ${counted} checked, the ${report.lookback_days} days to ${result.window.end}, at list price.`] + for (const one of result.skipped ?? []) lines.push(`Not checked: ${one.check} (${one.reason})`) if (report.findings.length === 0) { - lines.push("No savings found by the Serverless and Over-Sized checks.") + lines.push("No savings found.") return lines.join("\n") } lines.push( - `Total: ${range(report.total.monthly_min, report.total.monthly_max)} a month on ${report.total.warehouses} warehouse${report.total.warehouses === 1 ? "" : "s"}, each warehouse counted once at its largest saving.`, + `Total: ${range(report.total.monthly_min, report.total.monthly_max)} a month on ${report.total.resources} resource${report.total.resources === 1 ? "" : "s"}, each counted once at its largest saving.`, "", ) for (const finding of report.findings) { - lines.push(`${finding.warehouse_name} · ${finding.title} · ${range(finding.monthly_min, finding.monthly_max)} a month (${finding.severity})`) + lines.push( + `${finding.resource_name} (${finding.resource_type.replace("_", " ")}) · ${finding.title} · ${range(finding.monthly_min, finding.monthly_max)} a month (${finding.severity})`, + ) for (const line of evidenceLines(finding)) lines.push(` ${line}`) } + for (const option of report.what_if) { + lines.push("", `What ${option.resource_name} (now ${option.current_label}, ${dollars(option.current_monthly_cost)} a month) would cost instead:`) + for (const choice of option.choices) { + const fit = choice.fits ? "" : ` (too small: ${choice.cores} cores, its queries need ${choice.cores_needed.toFixed(0)})` + lines.push(` ${size(choice.label)}: ${dollars(choice.monthly_cost)} a month, saving ${dollars(choice.monthly_saving)}${fit}`) + } + } return lines.join("\n") } export const FinopsDatabricksSavingsTool = Tool.define("finops_databricks_savings", { description: - "Check a Databricks workspace's SQL warehouses for savings with Altimate's rules: Classic/Pro warehouses that would cost less as serverless, and warehouses larger than their queries need. Read-only; returns each finding with its monthly dollar range and the measured evidence.", + "Check a Databricks workspace for savings with Altimate's rules: SQL warehouses that would cost less as serverless or smaller, idle all-purpose clusters, and scheduled jobs running on all-purpose compute. Read-only queries over seven days of system tables (billing, compute, query history, jobs) on the connection's SQL warehouse, which Databricks starts if it is stopped. Returns each finding with its monthly dollar range and evidence, and what each flagged warehouse would cost at other sizes.", parameters: z.object({ warehouse: z .string() @@ -92,13 +125,17 @@ export const FinopsDatabricksSavingsTool = Tool.define("finops_databricks_saving } const total = result.report.total return { - title: `Databricks Savings: ${range(total.monthly_min, total.monthly_max)} a month on ${total.warehouses} warehouse(s)`, + title: `Databricks Savings: ${range(total.monthly_min, total.monthly_max)} a month on ${total.resources} resource(s)`, metadata: { success: true, window: result.window, warehouse: result.warehouse, ran_on: result.ran_on, warehouses_checked: result.warehouses_checked, + clusters_checked: result.clusters_checked, + jobs_checked: result.jobs_checked, + skipped: result.skipped, + warehouses: result.warehouses, report: result.report, } as SavingsMetadata, output: formatSavings(result), diff --git a/packages/opencode/test/altimate/finops-databricks-e2e.test.ts b/packages/opencode/test/altimate/finops-databricks-e2e.test.ts index fba4e58f89..417cf7a70a 100644 --- a/packages/opencode/test/altimate/finops-databricks-e2e.test.ts +++ b/packages/opencode/test/altimate/finops-databricks-e2e.test.ts @@ -215,7 +215,7 @@ describe.skipIf(!HAS_DATABRICKS)("Databricks FinOps Integration", () => { if (result.success) { expect(result.warehouses_checked).toBeGreaterThan(0) expect(Array.isArray(result.report!.findings)).toBe(true) - expect(result.report!.total.warehouses).toBe(new Set(result.report!.findings.map((f) => f.warehouse_id)).size) + expect(result.report!.total.resources).toBe(new Set(result.report!.findings.map((f) => `${f.resource_type}:${f.resource_id}`)).size) } }, 300000) }) diff --git a/packages/opencode/test/altimate/finops-databricks-savings.test.ts b/packages/opencode/test/altimate/finops-databricks-savings.test.ts index 8865901e20..5dc06b371c 100644 --- a/packages/opencode/test/altimate/finops-databricks-savings.test.ts +++ b/packages/opencode/test/altimate/finops-databricks-savings.test.ts @@ -3,10 +3,14 @@ import { initTool } from "./tool-fixture" import * as Dispatcher from "../../src/altimate/native/dispatcher" import * as Registry from "../../src/altimate/native/connections/registry" import { + DATABRICKS_CLUSTER_FACTS_SQL, + DATABRICKS_JOB_FACTS_SQL, DATABRICKS_WAREHOUSE_FACTS_SQL, databricksSavings, savingsWindow, + toClusterFacts, toFacts, + toJobFacts, } from "../../src/altimate/native/finops/databricks-savings" import { FinopsDatabricksSavingsTool } from "../../src/altimate/tools/finops-databricks-savings" import type { DatabricksSavingsResult } from "../../src/altimate/native/types" @@ -29,8 +33,9 @@ const REPORT: DatabricksSavingsResult = { { check_id: "wh_classic_pro_to_serverless", title: "Migrate Classic/Pro to Serverless", - warehouse_id: "a3688f261647f580", - warehouse_name: "Classic", + resource_type: "sql_warehouse", + resource_id: "a3688f261647f580", + resource_name: "Classic", severity: "critical", daily_min: 1.773465, daily_max: 2.216832, @@ -53,8 +58,9 @@ const REPORT: DatabricksSavingsResult = { { check_id: "wh_oversized", title: "Over-Sized Warehouse", - warehouse_id: "a3688f261647f580", - warehouse_name: "Classic", + resource_type: "sql_warehouse", + resource_id: "a3688f261647f580", + resource_name: "Classic", severity: "high", daily_min: 0.898177, daily_max: 1.57181, @@ -76,7 +82,20 @@ const REPORT: DatabricksSavingsResult = { }, }, ], - total: { monthly_min: 53.2, monthly_max: 66.5, warehouses: 1 }, + total: { monthly_min: 53.2, monthly_max: 66.5, resources: 1 }, + what_if: [ + { + resource_id: "a3688f261647f580", + resource_name: "Classic", + current_label: "SMALL (CLASSIC)", + current_monthly_cost: 101.05, + choices: [ + { label: "SERVERLESS", monthly_cost: 34.55, monthly_saving: 66.5, fits: true, cores: 0, cores_needed: 5.34 }, + { label: "X_SMALL", monthly_cost: 50.52, monthly_saving: 50.52, fits: true, cores: 16, cores_needed: 5.34 }, + { label: "2X_SMALL", monthly_cost: 33.68, monthly_saving: 67.37, fits: true, cores: 8, cores_needed: 5.34 }, + ], + }, + ], }, } @@ -88,9 +107,12 @@ describe("finops_databricks_savings tool", () => { const tool = await initTool(FinopsDatabricksSavingsTool) const result = await tool.execute({}, stubCtx()) - expect(result.title).toBe("Databricks Savings: $53.20–$66.50 a month on 1 warehouse(s)") - expect(result.output).toContain("Total: $53.20–$66.50 a month on 1 warehouse, each warehouse counted once") - expect(result.output).toContain("Classic · Migrate Classic/Pro to Serverless · $53.20–$66.50 a month (critical)") + expect(result.title).toBe("Databricks Savings: $53.20–$66.50 a month on 1 resource(s)") + expect(result.output).toContain("Total: $53.20–$66.50 a month on 1 resource, each counted once at its largest saving.") + expect(result.output).toContain("Classic (sql warehouse) · Migrate Classic/Pro to Serverless · $53.20–$66.50 a month (critical)") + // What the flagged warehouse would cost instead, for Claude to explain. + expect(result.output).toContain("What Classic (now SMALL (CLASSIC), $101 a month) would cost instead:") + expect(result.output).toContain(" X-SMALL: $50.52 a month, saving $50.52") expect(result.output).toContain("CLASSIC SMALL warehouse, 89% idle over 10.9 running hours (3,580 queries)") expect(result.output).toContain("2X-SMALL is enough: 12 → 4 DBU an hour") expect(result.metadata.report).toEqual(REPORT.report) @@ -100,11 +122,11 @@ describe("finops_databricks_savings tool", () => { test("no findings says so and still reports how many warehouses were checked", async () => { Dispatcher.register("finops.databricks_savings" as any, async () => ({ ...REPORT, - report: { lookback_days: 7, findings: [], total: { monthly_min: 0, monthly_max: 0, warehouses: 0 } }, + report: { lookback_days: 7, findings: [], total: { monthly_min: 0, monthly_max: 0, resources: 0 }, what_if: [] }, })) const tool = await initTool(FinopsDatabricksSavingsTool) const result = await tool.execute({}, stubCtx()) - expect(result.output).toContain("5 warehouses checked") + expect(result.output).toContain("5 SQL warehouses, 0 all-purpose clusters, 0 jobs checked") expect(result.output).toContain("No savings found") }) @@ -176,3 +198,62 @@ describe("databricks savings handler", () => { expect(result.error).toContain("/sql/1.0/warehouses/") }) }) + +describe("cluster and job findings", () => { + beforeEach(() => Dispatcher.reset()) + + test("each kind of finding gets its own evidence lines, and a skipped check is named", async () => { + Dispatcher.register("finops.databricks_savings" as any, async () => ({ + ...REPORT, + clusters_checked: 3, + jobs_checked: 4, + skipped: [{ check: "Job Running on All-Purpose Compute", reason: "[INSUFFICIENT_PERMISSIONS] system.lakeflow.jobs" }], + report: { + lookback_days: 7, + findings: [ + { + check_id: "ap_idle_cluster", + title: "Idle Cluster Detection", + resource_type: "cluster", + resource_id: "0101-abc", + resource_name: "shared-analytics", + severity: "critical", + daily_min: 25.5, + daily_max: 51, + monthly_min: 765, + monthly_max: 1530, + evidence: { kind: "idle_cluster", idle_pattern: "zombie", idle_ratio: 0.85, auto_termination_minutes: null, active_days: 5, avg_cpu_active_pct: 8, peak_cpu_active_pct: 30, total_cost: 700, wasted_cost: 595 }, + }, + ], + total: { monthly_min: 765, monthly_max: 1530, resources: 1 }, + what_if: [], + }, + })) + const tool = await initTool(FinopsDatabricksSavingsTool) + const result = await tool.execute({}, stubCtx()) + expect(result.output).toContain("5 SQL warehouses, 3 all-purpose clusters, 4 jobs checked") + expect(result.output).toContain("Not checked: Job Running on All-Purpose Compute ([INSUFFICIENT_PERMISSIONS] system.lakeflow.jobs)") + expect(result.output).toContain("shared-analytics (cluster) · Idle Cluster Detection · $765–$1,530 a month (critical)") + expect(result.output).toContain("All-purpose cluster 85% idle across 5 active days (zombie), average worker CPU 8%, peak 30%") + expect(result.output).toContain("Auto-termination off; $700.00 billed outside job runs over the window") + }) + + test("the cluster and job queries each take the window end and the warehouse id, in that order", () => { + for (const sql of [DATABRICKS_CLUSTER_FACTS_SQL, DATABRICKS_JOB_FACTS_SQL]) { + expect(sql.trim()).toMatch(/^WITH\b/) + expect(sql).not.toMatch(/;\s*\S/) + const markers = [...sql.matchAll(/\?/g)].map((m) => m.index!) + expect(markers.length).toBe(2) + expect(sql.slice(0, markers[0]!)).toContain("CAST(") + expect(sql.slice(markers[0]!, markers[1]!)).toContain("SELECT MAX(workspace_id) FROM system.compute.warehouses WHERE warehouse_id =") + } + }) + + test("cluster and job rows are coerced to the facts core reads", () => { + const cluster = toClusterFacts({ cluster_id: "c1", cluster_name: null, auto_termination_minutes: "120", total_node_minutes: 900n, active_util_days: "4", total_worker_minutes: 900, idle_node_minutes: 450, total_cost: "70.5" }) + expect(cluster).toMatchObject({ cluster_id: "c1", cluster_name: "c1", auto_termination_minutes: 120, total_node_minutes: 900, active_util_days: 4, total_cost: 70.5 }) + expect(toClusterFacts({ cluster_id: "c2" }).auto_termination_minutes).toBeNull() + const job = toJobFacts({ job_id: "j1", job_name: "nightly", trigger_type: null, run_count: 7n, job_share_of_cluster: 1.2, jobs_rate_per_dbu: null }) + expect(job).toMatchObject({ job_id: "j1", trigger_type: null, run_count: 7, job_share_of_cluster: 1, jobs_rate_per_dbu: null }) + }) +}) From 022b90cb4caf953e9265beee66fc7ed61059cb37 Mon Sep 17 00:00:00 2001 From: Shreyas Telkar Date: Tue, 17 Nov 2026 16:57:02 -0800 Subject: [PATCH 03/34] fix: hand core the job inventory's trigger; window, limit and blank-name fixes - The job query returns the inventory's trigger, whether the inventory lists the job, and the runs' trigger separately, for core to decide manual-only as Altimate's pipeline does. - The window ends at the latest 06:00 UTC that has passed, never in the future. - A query that fills the 10,000-row limit is named in `limited` and in the output. - A blank connection name falls back to the connection that runs through a SQL warehouse. - The handler takes the rules engine as an optional argument, so tests drive it end to end over a stubbed connection: a skipped check, the bind order, what reaches core, the row limit. - Says in the warehouse query why billing is read by whole usage days. Co-Authored-By: Claude Opus 5.5 --- .../native/finops/databricks-savings.ts | 50 +++++++--- .../opencode/src/altimate/native/types.ts | 2 + .../tools/finops-databricks-savings.ts | 3 + .../finops-databricks-savings.test.ts | 96 ++++++++++++++++++- 4 files changed, 133 insertions(+), 18 deletions(-) diff --git a/packages/opencode/src/altimate/native/finops/databricks-savings.ts b/packages/opencode/src/altimate/native/finops/databricks-savings.ts index 02954bd67e..a958420d16 100644 --- a/packages/opencode/src/altimate/native/finops/databricks-savings.ts +++ b/packages/opencode/src/altimate/native/finops/databricks-savings.ts @@ -18,6 +18,12 @@ const LOOKBACK_DAYS = 7 const DAY_MS = 86_400_000 +/** The window ends at 06:00 UTC, once the day before has been billed. */ +const WINDOW_END_MS = 6 * 3_600_000 + +/** Rows read per query; a query that fills it says so in the result. */ +const ROW_LIMIT = 10_000 + /** A SQL warehouse's id in a connection's HTTP path (`/sql/1.0/warehouses/`). */ const WAREHOUSE_PATH = /\/warehouses\/([0-9a-z]+)\/?$/i @@ -26,7 +32,9 @@ const WAREHOUSE_PATH = /\/warehouses\/([0-9a-z]+)\/?$/i * warehouse in the workspace of the warehouse named by the second. System tables hold * every workspace of the account, so each input is cut to that workspace's warehouses. * Billing that landed after the window's end is left out, so a past window gives the - * same facts on every run. + * same facts on every run. Billing is read as Altimate's opportunity pipeline reads it for + * these checks: whole usage days inside the window, ingested before its end day, which is + * the six full days before that day. */ export const DATABRICKS_WAREHOUSE_FACTS_SQL = ` WITH p AS ( @@ -266,7 +274,9 @@ inv AS ( QUALIFY ROW_NUMBER() OVER (PARTITION BY job_id ORDER BY change_time DESC) = 1 ) SELECT jb.job_id, COALESCE(inv.name, 'job ' || jb.job_id) AS job_name, - COALESCE(inv.trigger_type, runs.run_trigger) AS trigger_type, + inv.trigger_type AS trigger_type, + inv.job_id IS NOT NULL AS listed_in_job_inventory, + runs.run_trigger AS run_trigger_type, COALESCE(runs.run_count, 0) AS run_count, jb.job_total_cost, jb.job_all_purpose_cost, jb.job_all_purpose_dbus, CASE WHEN mc.cluster_id IS NULL OR COALESCE(ct.cost, 0) <= 0 THEN 1.0 ELSE mc.cost / ct.cost END AS job_share_of_cluster, @@ -280,7 +290,8 @@ WHERE jb.job_all_purpose_cost > 0 ORDER BY jb.job_all_purpose_cost DESC ` -type RulesEngine = (input: string) => DatabricksSavingsReport +/** Core's rules: a `DatabricksSavingsInput` as JSON in, the report out. */ +export type RulesEngine = (input: string) => DatabricksSavingsReport /** The installed core's rules, or null when its binary predates them. */ function rulesEngine(): RulesEngine | null { @@ -288,9 +299,9 @@ function rulesEngine(): RulesEngine | null { return typeof found === "function" ? (found as RulesEngine) : null } -/** The seven-day window the facts cover: it ends at 06:00 UTC on `end`, today's UTC date. */ +/** The seven-day window the facts cover: it ends at 06:00 UTC on `end`, the latest one that has passed. */ export function savingsWindow(nowMs: number): { start: string; end: string } { - const today = Math.floor(nowMs / DAY_MS) * DAY_MS + const today = Math.floor((nowMs - WINDOW_END_MS) / DAY_MS) * DAY_MS const day = (ms: number) => new Date(ms).toISOString().slice(0, 10) return { start: day(today - LOOKBACK_DAYS * DAY_MS), end: day(today) } } @@ -352,6 +363,8 @@ export function toJobFacts(row: Record) { job_id: id, job_name: String(row.job_name ?? `job ${id}`), trigger_type: row.trigger_type == null ? null : String(row.trigger_type), + listed_in_job_inventory: [true, 1, "true"].includes(row.listed_in_job_inventory as never), + run_trigger_type: row.run_trigger_type == null ? null : String(row.run_trigger_type), run_count: Math.round(num(row.run_count)), job_total_cost: num(row.job_total_cost), job_all_purpose_cost: num(row.job_all_purpose_cost), @@ -374,12 +387,16 @@ function preferredConnection(): string | undefined { })?.name } -export async function databricksSavings(params: DatabricksSavingsParams): Promise { +/** `rules` defaults to the installed core's; tests pass their own. */ +export async function databricksSavings( + params: DatabricksSavingsParams, + rules: RulesEngine | null = rulesEngine(), +): Promise { const window = savingsWindow(params.now_ms ?? Date.now()) const failed = (error: string): DatabricksSavingsResult => ({ success: false, window, error }) const resolved = resolveFinopsWarehouse({ - requested: params.warehouse ?? preferredConnection(), + requested: params.warehouse?.trim() || preferredConnection(), supportedTypes: SUPPORTED_TYPES, operationName: "The Databricks savings check", }) @@ -395,7 +412,6 @@ export async function databricksSavings(params: DatabricksSavingsParams): Promis ) } - const rules = rulesEngine() if (!rules) { return failed("This altimate-code build's altimate-core has no Databricks savings rules. Update altimate-code, then try again.") } @@ -403,22 +419,29 @@ export async function databricksSavings(params: DatabricksSavingsParams): Promis try { const connector = await Registry.get(resolved.warehouse) const binds = [window.end, runsOn] - const warehouses = records(await connector.execute(DATABRICKS_WAREHOUSE_FACTS_SQL, 10_000, binds)) + // A query that fills the row limit reads only its first rows; the result names it. + const limited: string[] = [] + const read = async (what: string, sql: string) => { + const result = await connector.execute(sql, ROW_LIMIT, binds) + if (result.truncated) limited.push(what) + return records(result) + } + const warehouses = await read("SQL warehouses", DATABRICKS_WAREHOUSE_FACTS_SQL) if (warehouses.length === 0) { return failed(`No SQL warehouses were found for warehouse ${runsOn} in system.compute.warehouses.`) } // A table the connection may not read stops only its own check; the rest still report. const skipped: { check: string; reason: string }[] = [] - const optional = async (check: string, sql: string) => { + const optional = async (check: string, what: string, sql: string) => { try { - return records(await connector.execute(sql, 10_000, binds)) + return await read(what, sql) } catch (e) { skipped.push({ check, reason: (e instanceof Error ? e.message : String(e)).split("\n")[0]!.slice(0, 300) }) return [] } } - const clusters = await optional("Idle Cluster Detection", DATABRICKS_CLUSTER_FACTS_SQL) - const jobs = await optional("Job Running on All-Purpose Compute", DATABRICKS_JOB_FACTS_SQL) + const clusters = await optional("Idle Cluster Detection", "all-purpose clusters", DATABRICKS_CLUSTER_FACTS_SQL) + const jobs = await optional("Job Running on All-Purpose Compute", "jobs", DATABRICKS_JOB_FACTS_SQL) const report = rules( JSON.stringify({ lookback_days: LOOKBACK_DAYS, @@ -436,6 +459,7 @@ export async function databricksSavings(params: DatabricksSavingsParams): Promis clusters_checked: clusters.length, jobs_checked: jobs.length, skipped, + limited, // What each warehouse is and costs now, so a later run can tell what changed. warehouses: warehouses.map((row) => ({ id: String(row.warehouse_id ?? ""), diff --git a/packages/opencode/src/altimate/native/types.ts b/packages/opencode/src/altimate/native/types.ts index 2643b79008..bcad388e55 100644 --- a/packages/opencode/src/altimate/native/types.ts +++ b/packages/opencode/src/altimate/native/types.ts @@ -689,6 +689,8 @@ export interface DatabricksSavingsResult { jobs_checked?: number /** Checks whose system tables the connection could not read. */ skipped?: { check: string; reason: string }[] + /** What a query read only the first 10,000 of, e.g. `jobs`. */ + limited?: string[] /** Each SQL warehouse as it is now, and its cost over the window. */ warehouses?: { id: string; name: string; type: string; size: string | null; window_cost: number }[] report?: DatabricksSavingsReport diff --git a/packages/opencode/src/altimate/tools/finops-databricks-savings.ts b/packages/opencode/src/altimate/tools/finops-databricks-savings.ts index ec3185660f..9eb9db88c6 100644 --- a/packages/opencode/src/altimate/tools/finops-databricks-savings.ts +++ b/packages/opencode/src/altimate/tools/finops-databricks-savings.ts @@ -14,6 +14,7 @@ type SavingsMetadata = { clusters_checked?: number jobs_checked?: number skipped?: { check: string; reason: string }[] + limited?: string[] warehouses?: { id: string; name: string; type: string; size: string | null; window_cost: number }[] report?: DatabricksSavingsReport error?: string @@ -79,6 +80,7 @@ export function formatSavings(result: DatabricksSavingsResult): string { ].join(", ") const lines: string[] = [`Databricks savings: ${counted} checked, the ${report.lookback_days} days to ${result.window.end}, at list price.`] for (const one of result.skipped ?? []) lines.push(`Not checked: ${one.check} (${one.reason})`) + for (const what of result.limited ?? []) lines.push(`Only the first 10,000 ${what} were read; any beyond them were not checked.`) if (report.findings.length === 0) { lines.push("No savings found.") return lines.join("\n") @@ -135,6 +137,7 @@ export const FinopsDatabricksSavingsTool = Tool.define("finops_databricks_saving clusters_checked: result.clusters_checked, jobs_checked: result.jobs_checked, skipped: result.skipped, + limited: result.limited, warehouses: result.warehouses, report: result.report, } as SavingsMetadata, diff --git a/packages/opencode/test/altimate/finops-databricks-savings.test.ts b/packages/opencode/test/altimate/finops-databricks-savings.test.ts index 5dc06b371c..e84de7c8eb 100644 --- a/packages/opencode/test/altimate/finops-databricks-savings.test.ts +++ b/packages/opencode/test/altimate/finops-databricks-savings.test.ts @@ -1,4 +1,4 @@ -import { afterEach, beforeEach, describe, expect, test } from "bun:test" +import { afterEach, beforeEach, describe, expect, spyOn, test } from "bun:test" import { initTool } from "./tool-fixture" import * as Dispatcher from "../../src/altimate/native/dispatcher" import * as Registry from "../../src/altimate/native/connections/registry" @@ -12,7 +12,7 @@ import { toFacts, toJobFacts, } from "../../src/altimate/native/finops/databricks-savings" -import { FinopsDatabricksSavingsTool } from "../../src/altimate/tools/finops-databricks-savings" +import { FinopsDatabricksSavingsTool, formatSavings } from "../../src/altimate/tools/finops-databricks-savings" import type { DatabricksSavingsResult } from "../../src/altimate/native/types" function stubCtx(): any { @@ -147,8 +147,11 @@ describe("finops_databricks_savings tool", () => { describe("databricks savings handler", () => { afterEach(() => Registry.reset()) - test("the window is the seven days to 06:00 UTC today", () => { + test("the window is the seven days to the latest 06:00 UTC that has passed", () => { expect(savingsWindow(Date.parse("2026-11-12T10:00:00Z"))).toEqual({ start: "2026-11-05", end: "2026-11-12" }) + expect(savingsWindow(Date.parse("2026-11-12T06:00:00Z"))).toEqual({ start: "2026-11-05", end: "2026-11-12" }) + // Before 06:00 the day before is not billed yet, so the window still ends the day before. + expect(savingsWindow(Date.parse("2026-11-12T05:59:00Z"))).toEqual({ start: "2026-11-04", end: "2026-11-11" }) }) test("the facts query is one read-only statement taking the window end and the warehouse id, in that order", () => { @@ -253,7 +256,90 @@ describe("cluster and job findings", () => { const cluster = toClusterFacts({ cluster_id: "c1", cluster_name: null, auto_termination_minutes: "120", total_node_minutes: 900n, active_util_days: "4", total_worker_minutes: 900, idle_node_minutes: 450, total_cost: "70.5" }) expect(cluster).toMatchObject({ cluster_id: "c1", cluster_name: "c1", auto_termination_minutes: 120, total_node_minutes: 900, active_util_days: 4, total_cost: 70.5 }) expect(toClusterFacts({ cluster_id: "c2" }).auto_termination_minutes).toBeNull() - const job = toJobFacts({ job_id: "j1", job_name: "nightly", trigger_type: null, run_count: 7n, job_share_of_cluster: 1.2, jobs_rate_per_dbu: null }) - expect(job).toMatchObject({ job_id: "j1", trigger_type: null, run_count: 7, job_share_of_cluster: 1, jobs_rate_per_dbu: null }) + const job = toJobFacts({ job_id: "j1", job_name: "nightly", trigger_type: null, listed_in_job_inventory: true, run_trigger_type: "ONE_TIME", run_count: 7n, job_share_of_cluster: 1.2, jobs_rate_per_dbu: null }) + expect(job).toMatchObject({ job_id: "j1", trigger_type: null, listed_in_job_inventory: true, run_trigger_type: "ONE_TIME", run_count: 7, job_share_of_cluster: 1, jobs_rate_per_dbu: null }) + // A job the inventory does not list arrives with false, or with nothing at all. + expect(toJobFacts({ job_id: "j2", listed_in_job_inventory: false }).listed_in_job_inventory).toBe(false) + expect(toJobFacts({ job_id: "j3" })).toMatchObject({ listed_in_job_inventory: false, run_trigger_type: null }) + }) + + test("the job query hands the inventory's trigger and the runs' trigger to core separately", () => { + expect(DATABRICKS_JOB_FACTS_SQL).toContain("inv.trigger_type AS trigger_type") + expect(DATABRICKS_JOB_FACTS_SQL).toContain("inv.job_id IS NOT NULL AS listed_in_job_inventory") + expect(DATABRICKS_JOB_FACTS_SQL).toContain("runs.run_trigger AS run_trigger_type") + expect(DATABRICKS_JOB_FACTS_SQL).not.toContain("COALESCE(inv.trigger_type") + }) +}) + +describe("databricks savings handler over a stubbed connection", () => { + const WAREHOUSE_ROW = { warehouse_id: "abc123", warehouse_name: "BI", warehouse_type: "PRO", warehouse_size: "SMALL", total_cost: 27.5 } + const JOB_ROW = { job_id: "j1", job_name: "nightly", trigger_type: null, listed_in_job_inventory: true, run_trigger_type: "ONE_TIME", run_count: 9 } + const table = (row: Record, truncated = false) => ({ columns: Object.keys(row), rows: [Object.values(row)], truncated }) + let calls: { sql: string; limit: number; binds: unknown[] }[] + let sent: any + + beforeEach(() => { + calls = [] + sent = undefined + Registry.setConfigs({ + dbx: { type: "databricks", server_hostname: "dbc-0.cloud.databricks.com", http_path: "/sql/1.0/warehouses/abc123", access_token: "x" }, + }) + }) + afterEach(() => { + Registry.reset() + }) + + const rules = (input: string) => { + sent = JSON.parse(input) + return { lookback_days: 7, findings: [], total: { monthly_min: 0, monthly_max: 0, resources: 0 }, what_if: [] } + } + + const connect = (answer: (sql: string) => unknown) => + spyOn(Registry, "get").mockResolvedValue({ + execute: async (sql: string, limit: number, binds: unknown[]) => { + calls.push({ sql, limit, binds }) + return answer(sql) + }, + } as any) + + test("a check whose tables the connection cannot read is named as skipped; the others still reach core", async () => { + const spy = connect((sql) => { + if (sql === DATABRICKS_CLUSTER_FACTS_SQL) throw new Error("[INSUFFICIENT_PERMISSIONS] system.compute.node_timeline\nstack") + return sql === DATABRICKS_WAREHOUSE_FACTS_SQL ? table(WAREHOUSE_ROW) : table(JOB_ROW) + }) + const result = await databricksSavings({ warehouse: "dbx", now_ms: Date.parse("2026-11-18T10:00:00Z") }, rules) + spy.mockRestore() + expect(result.success).toBe(true) + expect(result.skipped).toEqual([{ check: "Idle Cluster Detection", reason: "[INSUFFICIENT_PERMISSIONS] system.compute.node_timeline" }]) + expect([result.warehouses_checked, result.clusters_checked, result.jobs_checked]).toEqual([1, 0, 1]) + // Every query takes the window end, then the warehouse the connection runs on. + expect(calls.map((one) => one.sql)).toEqual([DATABRICKS_WAREHOUSE_FACTS_SQL, DATABRICKS_CLUSTER_FACTS_SQL, DATABRICKS_JOB_FACTS_SQL]) + for (const one of calls) expect(one.binds).toEqual(["2026-11-18", "abc123"]) + expect(sent.lookback_days).toBe(7) + expect(sent.warehouses[0]).toMatchObject({ warehouse_id: "abc123", warehouse_size: "SMALL", total_cost: 27.5 }) + expect(sent.clusters).toEqual([]) + expect(sent.jobs[0]).toMatchObject({ job_id: "j1", trigger_type: null, listed_in_job_inventory: true, run_trigger_type: "ONE_TIME" }) + expect(result.limited).toEqual([]) + }) + + test("a query that fills the row limit is named, and the tool says so", async () => { + const spy = connect((sql) => (sql === DATABRICKS_WAREHOUSE_FACTS_SQL ? table(WAREHOUSE_ROW) : sql === DATABRICKS_JOB_FACTS_SQL ? table(JOB_ROW, true) : table({ cluster_id: "c1" }))) + const result = await databricksSavings({ warehouse: "dbx", now_ms: Date.parse("2026-11-18T10:00:00Z") }, rules) + spy.mockRestore() + expect(calls.every((one) => one.limit === 10_000)).toBe(true) + expect(result.limited).toEqual(["jobs"]) + expect(formatSavings(result)).toContain("Only the first 10,000 jobs were read; any beyond them were not checked.") + }) + + test("a blank connection name falls back to the connection that runs through a SQL warehouse", async () => { + Registry.setConfigs({ + cluster: { type: "databricks", server_hostname: "dbc-0.cloud.databricks.com", http_path: "/sql/protocolv1/o/1/0101-abc", access_token: "x" }, + dbx: { type: "databricks", server_hostname: "dbc-0.cloud.databricks.com", http_path: "/sql/1.0/warehouses/abc123", access_token: "x" }, + }) + const spy = connect((sql) => (sql === DATABRICKS_WAREHOUSE_FACTS_SQL ? table(WAREHOUSE_ROW) : { columns: [], rows: [], truncated: false })) + const result = await databricksSavings({ warehouse: " ", now_ms: Date.parse("2026-11-18T10:00:00Z") }, rules) + spy.mockRestore() + expect(result.success).toBe(true) + expect(result.warehouse).toBe("dbx") }) }) From 44ed9d82a6fa3bed71364a913a625ed7b7d1b407 Mon Sep 17 00:00:00 2001 From: Shreyas Telkar Date: Tue, 17 Nov 2026 17:19:41 -0800 Subject: [PATCH 04/34] fix: keep the warehouse findings when core refuses cluster or job facts - If core rejects the input over a cluster or job value, the warehouses are re-checked alone and those two checks are named as skipped with the reason. - Clusters are read costliest first, so a row limit keeps the ones that matter. - Says in the query that uptime counts only time between events in the window, as the pipeline does. Co-Authored-By: Claude Opus 5.5 --- .../native/finops/databricks-savings.ts | 32 +++++++++++++------ .../finops-databricks-savings.test.ts | 19 +++++++++++ 2 files changed, 42 insertions(+), 9 deletions(-) diff --git a/packages/opencode/src/altimate/native/finops/databricks-savings.ts b/packages/opencode/src/altimate/native/finops/databricks-savings.ts index a958420d16..6e473e49e1 100644 --- a/packages/opencode/src/altimate/native/finops/databricks-savings.ts +++ b/packages/opencode/src/altimate/native/finops/databricks-savings.ts @@ -57,6 +57,8 @@ ev AS ( unix_timestamp(LEAD(e.event_time) OVER (PARTITION BY e.workspace_id, e.warehouse_id ORDER BY e.event_time)) - unix_timestamp(e.event_time) AS dur_sec FROM system.compute.warehouse_events e CROSS JOIN w + -- As in Altimate's pipeline, uptime counts only the time between two events inside the window: + -- a warehouse that runs all week without a scale or stop event reads no uptime. WHERE e.event_time >= w.rs AND e.event_time < w.r AND e.workspace_id IN (SELECT workspace_id FROM inv) AND e.warehouse_id IN (SELECT warehouse_id FROM inv) ), @@ -214,7 +216,7 @@ FROM inv JOIN util ON util.cluster_id = inv.cluster_id LEFT JOIN bill ON bill.cluster_id = inv.cluster_id WHERE inv.delete_time IS NULL AND inv.cluster_source IN ('UI', 'API') -ORDER BY inv.cluster_name +ORDER BY total_cost DESC, inv.cluster_name ` /** @@ -442,14 +444,26 @@ export async function databricksSavings( } const clusters = await optional("Idle Cluster Detection", "all-purpose clusters", DATABRICKS_CLUSTER_FACTS_SQL) const jobs = await optional("Job Running on All-Purpose Compute", "jobs", DATABRICKS_JOB_FACTS_SQL) - const report = rules( - JSON.stringify({ - lookback_days: LOOKBACK_DAYS, - warehouses: warehouses.map(toFacts), - clusters: clusters.map(toClusterFacts), - jobs: jobs.map(toJobFacts), - }), - ) + const input = { + lookback_days: LOOKBACK_DAYS, + warehouses: warehouses.map(toFacts), + clusters: clusters.map(toClusterFacts), + jobs: jobs.map(toJobFacts), + } + let report: DatabricksSavingsReport + try { + report = rules(JSON.stringify(input)) + } catch (e) { + // Core refuses the whole input over one bad value. If the warehouses alone pass, the + // cluster and job checks are named as skipped instead of losing every finding. + if (input.clusters.length === 0 && input.jobs.length === 0) throw e + report = rules(JSON.stringify({ ...input, clusters: [], jobs: [] })) + const reason = `core refused their facts: ${(e instanceof Error ? e.message : String(e)).split("\n")[0]!.slice(0, 240)}` + if (input.clusters.length > 0) skipped.push({ check: "Idle Cluster Detection", reason }) + if (input.jobs.length > 0) skipped.push({ check: "Job Running on All-Purpose Compute", reason }) + clusters.length = 0 + jobs.length = 0 + } return { success: true, window, diff --git a/packages/opencode/test/altimate/finops-databricks-savings.test.ts b/packages/opencode/test/altimate/finops-databricks-savings.test.ts index e84de7c8eb..f7339989f7 100644 --- a/packages/opencode/test/altimate/finops-databricks-savings.test.ts +++ b/packages/opencode/test/altimate/finops-databricks-savings.test.ts @@ -331,6 +331,25 @@ describe("databricks savings handler over a stubbed connection", () => { expect(formatSavings(result)).toContain("Only the first 10,000 jobs were read; any beyond them were not checked.") }) + test("when core refuses the cluster or job facts, the warehouse findings still report and those checks are named", async () => { + const spy = connect((sql) => (sql === DATABRICKS_WAREHOUSE_FACTS_SQL ? table(WAREHOUSE_ROW) : sql === DATABRICKS_JOB_FACTS_SQL ? table(JOB_ROW) : table({ cluster_id: "c1" }))) + const calls: any[] = [] + const picky = (input: string) => { + const parsed = JSON.parse(input) + calls.push(parsed) + if (parsed.clusters.length > 0 || parsed.jobs.length > 0) throw new Error("invalid input: jobs[0].job_total_cost must not be negative") + return rules(input) + } + const result = await databricksSavings({ warehouse: "dbx", now_ms: Date.parse("2026-11-18T10:00:00Z") }, picky) + spy.mockRestore() + expect(result.success).toBe(true) + expect(calls.length).toBe(2) + expect(sent.warehouses[0].warehouse_id).toBe("abc123") + expect(result.skipped?.map((one) => one.check)).toEqual(["Idle Cluster Detection", "Job Running on All-Purpose Compute"]) + expect(result.skipped?.[0]?.reason).toContain("core refused their facts: invalid input: jobs[0].job_total_cost") + expect([result.clusters_checked, result.jobs_checked]).toEqual([0, 0]) + }) + test("a blank connection name falls back to the connection that runs through a SQL warehouse", async () => { Registry.setConfigs({ cluster: { type: "databricks", server_hostname: "dbc-0.cloud.databricks.com", http_path: "/sql/protocolv1/o/1/0101-abc", access_token: "x" }, From bcb5af225816f4b85427878ec47c7e130188b8b1 Mon Sep 17 00:00:00 2001 From: Shreyas Telkar Date: Tue, 17 Nov 2026 17:35:15 -0800 Subject: [PATCH 05/34] feat: read every workspace the connection's system tables cover - The three queries drop the one-workspace filter and take only the window end; each row names its workspace. Job ids repeat across workspaces, so every job key pairs workspace and id, and the jobs-compute rate is per workspace, as in Altimate's pipeline. - Findings name their workspace in the output; the older `/sql/1.0/endpoints/` path is accepted. Co-Authored-By: Claude Opus 5.5 --- .../data-engineering/tools/finops-tools.md | 2 +- .../native/finops/databricks-savings.ts | 125 ++++++++---------- .../opencode/src/altimate/native/types.ts | 4 +- .../tools/finops-databricks-savings.ts | 6 +- .../finops-databricks-savings.test.ts | 59 +++++++-- 5 files changed, 112 insertions(+), 84 deletions(-) diff --git a/docs/docs/data-engineering/tools/finops-tools.md b/docs/docs/data-engineering/tools/finops-tools.md index 54b9bc520f..0212976744 100644 --- a/docs/docs/data-engineering/tools/finops-tools.md +++ b/docs/docs/data-engineering/tools/finops-tools.md @@ -180,7 +180,7 @@ Recommendations: ## finops_databricks_savings -Check a Databricks workspace for savings with Altimate's rules, on a Databricks connection whose `http_path` is a SQL warehouse (`/sql/1.0/warehouses/`). It reads seven days of system tables (`system.billing`, `system.compute`, `system.query`, `system.lakeflow`) for the workspace that warehouse belongs to, on that warehouse, which Databricks starts if it is stopped. Nothing is written. Four checks: +Check a Databricks workspace for savings with Altimate's rules, on a Databricks connection whose `http_path` is a SQL warehouse (`/sql/1.0/warehouses/`). It reads seven days of system tables (`system.billing`, `system.compute`, `system.query`, `system.lakeflow`) on that warehouse, which Databricks starts if it is stopped. System tables cover every workspace of the account they can see, so one connection checks all of them; each finding names its workspace. Nothing is written. Four checks: - **Migrate Classic/Pro to Serverless:** a Classic or Pro warehouse that bills for time it sits idle. - **Over-Sized Warehouse:** a warehouse whose queries never use the cores it pays for. diff --git a/packages/opencode/src/altimate/native/finops/databricks-savings.ts b/packages/opencode/src/altimate/native/finops/databricks-savings.ts index 6e473e49e1..6ccb13e33e 100644 --- a/packages/opencode/src/altimate/native/finops/databricks-savings.ts +++ b/packages/opencode/src/altimate/native/finops/databricks-savings.ts @@ -24,13 +24,13 @@ const WINDOW_END_MS = 6 * 3_600_000 /** Rows read per query; a query that fills it says so in the result. */ const ROW_LIMIT = 10_000 -/** A SQL warehouse's id in a connection's HTTP path (`/sql/1.0/warehouses/`). */ -const WAREHOUSE_PATH = /\/warehouses\/([0-9a-z]+)\/?$/i +/** A SQL warehouse's id in a connection's HTTP path (`/sql/1.0/warehouses/`, or the older `/sql/1.0/endpoints/`). */ +const WAREHOUSE_PATH = /\/(?:warehouses|endpoints)\/([0-9a-z]+)\/?$/i /** - * Per-warehouse usage over the seven days to 06:00 UTC on the first parameter, for every - * warehouse in the workspace of the warehouse named by the second. System tables hold - * every workspace of the account, so each input is cut to that workspace's warehouses. + * Per-warehouse usage over the seven days to 06:00 UTC on the parameter, for every warehouse of + * every workspace the connection's system tables cover (as Altimate's pipeline reads them), each + * row naming its workspace. * Billing that landed after the window's end is left out, so a past window gives the * same facts on every run. Billing is read as Altimate's opportunity pipeline reads it for * these checks: whole usage days inside the window, ingested before its end day, which is @@ -49,7 +49,6 @@ w AS ( inv AS ( SELECT workspace_id, warehouse_id, warehouse_name, warehouse_type, warehouse_size, delete_time FROM system.compute.warehouses - WHERE workspace_id = (SELECT MAX(workspace_id) FROM system.compute.warehouses WHERE warehouse_id = ?) QUALIFY ROW_NUMBER() OVER (PARTITION BY workspace_id, warehouse_id ORDER BY change_time DESC) = 1 ), ev AS ( @@ -124,7 +123,7 @@ type_prices AS ( SELECT MAX(CASE WHEN sku_name LIKE '%SERVERLESS_SQL_COMPUTE%' THEN price END) AS serverless_price_per_dbu FROM bill_rows ) -SELECT inv.warehouse_id, inv.warehouse_name, inv.warehouse_type, inv.warehouse_size, +SELECT inv.workspace_id, inv.warehouse_id, inv.warehouse_name, inv.warehouse_type, inv.warehouse_size, COALESCE(up.wall_clock_uptime_sec, 0.0) AS wall_clock_uptime_sec, COALESCE(up.cluster_weighted_uptime_sec, 0.0) AS cluster_weighted_uptime_sec, CASE WHEN COALESCE(up.wall_clock_uptime_sec, 0) <= 0 THEN 0.0 @@ -149,9 +148,6 @@ WHERE inv.delete_time IS NULL ORDER BY inv.warehouse_name ` -/** The workspace of the warehouse the connection runs on: every input is cut to it. */ -const WORKSPACE_OF_WAREHOUSE = "(SELECT MAX(workspace_id) FROM system.compute.warehouses WHERE warehouse_id = ?)" - /** List price of a usage row: the price in force when the usage started. */ const LIST_PRICE_JOIN = `LEFT JOIN system.billing.list_prices lp ON u.sku_name = lp.sku_name AND u.cloud = lp.cloud @@ -159,135 +155,126 @@ const LIST_PRICE_JOIN = `LEFT JOIN system.billing.list_prices lp AND (lp.price_end_time IS NULL OR u.usage_start_time < lp.price_end_time)` /** - * Per all-purpose cluster (created from the UI or API, not deleted) of the workspace: worker node - * minutes, active days, idle worker minutes (CPU under 5%, CPU wait under 5%, under 10 MB of network - * a minute), CPU averages, and the all-purpose cost billed outside job runs. Seven full days before - * the window end. Parameters: the window end, then the warehouse id. + * Per all-purpose cluster (created from the UI or API, not deleted) of every workspace the + * connection's system tables cover: worker node minutes, active days, idle worker minutes (CPU + * under 5%, CPU wait under 5%, under 10 MB of network a minute), CPU averages, and the + * all-purpose cost billed outside job runs. Seven full days before the window end, the parameter. */ export const DATABRICKS_CLUSTER_FACTS_SQL = ` WITH p AS ( SELECT CAST(? AS DATE) AS we_date ), -ws AS ( - SELECT ${WORKSPACE_OF_WAREHOUSE} AS workspace_id -), inv AS ( - SELECT cluster_id, cluster_name, auto_termination_minutes, cluster_source, delete_time + SELECT workspace_id, cluster_id, cluster_name, auto_termination_minutes, cluster_source, delete_time FROM system.compute.clusters - WHERE workspace_id = (SELECT workspace_id FROM ws) - QUALIFY ROW_NUMBER() OVER (PARTITION BY cluster_id ORDER BY change_time DESC) = 1 + QUALIFY ROW_NUMBER() OVER (PARTITION BY workspace_id, cluster_id ORDER BY change_time DESC) = 1 ), nodes AS ( - SELECT n.cluster_id, n.start_time, n.end_time, + SELECT n.workspace_id, n.cluster_id, n.start_time, n.end_time, COALESCE(n.cpu_user_percent, 0) + COALESCE(n.cpu_system_percent, 0) AS cpu_active, COALESCE(n.cpu_wait_percent, 0) AS cpu_wait, COALESCE(n.network_sent_bytes, 0) + COALESCE(n.network_received_bytes, 0) AS network_bytes FROM system.compute.node_timeline n CROSS JOIN p - WHERE n.workspace_id = (SELECT workspace_id FROM ws) - AND n.start_time >= CAST(DATE_SUB(p.we_date, 7) AS TIMESTAMP) AND n.start_time < CAST(p.we_date AS TIMESTAMP) + WHERE n.start_time >= CAST(DATE_SUB(p.we_date, 7) AS TIMESTAMP) AND n.start_time < CAST(p.we_date AS TIMESTAMP) AND NOT COALESCE(n.driver, false) AND n.cluster_id IN (SELECT cluster_id FROM inv) ), util AS ( - SELECT cluster_id, + SELECT workspace_id, cluster_id, COUNT(DISTINCT CAST(start_time AS DATE)) AS active_util_days, SUM(timestampdiff(MINUTE, start_time, end_time)) AS total_node_minutes, COUNT(*) AS total_worker_minutes, COUNT_IF(cpu_active < 5.0 AND cpu_wait < 5.0 AND network_bytes < 10485760) AS idle_node_minutes, AVG(cpu_active) AS avg_cpu_active_pct, MAX(cpu_active) AS peak_cpu_active_pct - FROM nodes GROUP BY cluster_id + FROM nodes GROUP BY workspace_id, cluster_id ), bill AS ( - SELECT u.usage_metadata.cluster_id AS cluster_id, SUM(u.usage_quantity * lp.pricing.default) AS total_cost + SELECT u.workspace_id, u.usage_metadata.cluster_id AS cluster_id, SUM(u.usage_quantity * lp.pricing.default) AS total_cost FROM system.billing.usage u CROSS JOIN p ${LIST_PRICE_JOIN} - WHERE u.workspace_id = (SELECT workspace_id FROM ws) - AND u.billing_origin_product = 'ALL_PURPOSE' + WHERE u.billing_origin_product = 'ALL_PURPOSE' AND u.usage_metadata.cluster_id IS NOT NULL AND u.usage_metadata.job_id IS NULL AND u.usage_date >= DATE_SUB(p.we_date, 7) AND u.usage_date < p.we_date - GROUP BY u.usage_metadata.cluster_id + GROUP BY u.workspace_id, u.usage_metadata.cluster_id ) -SELECT inv.cluster_id, inv.cluster_name, inv.auto_termination_minutes, +SELECT inv.workspace_id, inv.cluster_id, inv.cluster_name, inv.auto_termination_minutes, util.total_node_minutes, util.active_util_days, util.total_worker_minutes, util.idle_node_minutes, util.avg_cpu_active_pct, util.peak_cpu_active_pct, COALESCE(bill.total_cost, 0.0) AS total_cost FROM inv -JOIN util ON util.cluster_id = inv.cluster_id -LEFT JOIN bill ON bill.cluster_id = inv.cluster_id +JOIN util ON util.workspace_id = inv.workspace_id AND util.cluster_id = inv.cluster_id +LEFT JOIN bill ON bill.workspace_id = inv.workspace_id AND bill.cluster_id = inv.cluster_id WHERE inv.delete_time IS NULL AND inv.cluster_source IN ('UI', 'API') -ORDER BY total_cost DESC, inv.cluster_name +ORDER BY COALESCE(bill.total_cost, 0.0) DESC, inv.cluster_name ` /** - * Per job of the workspace that billed in the window: its cost on every product and on all-purpose - * compute, its share of the all-purpose cluster it used most, its runs and trigger, and the - * workspace's jobs-compute rate. Seven full days before the window end. Parameters: the window end, - * then the warehouse id. + * Per job of every workspace the connection's system tables cover that billed in the window: its + * cost on every product and on all-purpose compute, its share of the all-purpose cluster it used + * most, its runs and trigger, and its workspace's jobs-compute rate. Job ids repeat across + * workspaces, so every key pairs the workspace with the id. Seven full days before the window end, + * the parameter. */ export const DATABRICKS_JOB_FACTS_SQL = ` WITH p AS ( SELECT CAST(? AS DATE) AS we_date ), -ws AS ( - SELECT ${WORKSPACE_OF_WAREHOUSE} AS workspace_id -), bill AS ( - SELECT u.usage_metadata.job_id AS job_id, u.usage_metadata.cluster_id AS cluster_id, u.billing_origin_product AS product, - u.usage_quantity AS dbus, u.usage_quantity * lp.pricing.default AS cost + SELECT u.workspace_id, u.usage_metadata.job_id AS job_id, u.usage_metadata.cluster_id AS cluster_id, + u.billing_origin_product AS product, u.usage_quantity AS dbus, u.usage_quantity * lp.pricing.default AS cost FROM system.billing.usage u CROSS JOIN p ${LIST_PRICE_JOIN} - WHERE u.workspace_id = (SELECT workspace_id FROM ws) - AND u.usage_date >= DATE_SUB(p.we_date, 7) AND u.usage_date < p.we_date + WHERE u.usage_date >= DATE_SUB(p.we_date, 7) AND u.usage_date < p.we_date ), job_bill AS ( - SELECT job_id, + SELECT workspace_id, job_id, SUM(COALESCE(cost, 0)) AS job_total_cost, SUM(CASE WHEN product = 'ALL_PURPOSE' THEN COALESCE(cost, 0) ELSE 0 END) AS job_all_purpose_cost, SUM(CASE WHEN product = 'ALL_PURPOSE' THEN dbus ELSE 0 END) AS job_all_purpose_dbus - FROM bill WHERE job_id IS NOT NULL AND job_id <> '' GROUP BY job_id + FROM bill WHERE job_id IS NOT NULL AND job_id <> '' GROUP BY workspace_id, job_id ), job_cluster AS ( - SELECT job_id, cluster_id, SUM(COALESCE(cost, 0)) AS cost + SELECT workspace_id, job_id, cluster_id, SUM(COALESCE(cost, 0)) AS cost FROM bill WHERE product = 'ALL_PURPOSE' AND job_id IS NOT NULL AND job_id <> '' AND cluster_id IS NOT NULL - GROUP BY job_id, cluster_id + GROUP BY workspace_id, job_id, cluster_id ), main_cluster AS ( - SELECT job_id, cluster_id, cost FROM job_cluster - QUALIFY ROW_NUMBER() OVER (PARTITION BY job_id ORDER BY cost DESC, cluster_id) = 1 + SELECT workspace_id, job_id, cluster_id, cost FROM job_cluster + QUALIFY ROW_NUMBER() OVER (PARTITION BY workspace_id, job_id ORDER BY cost DESC, cluster_id) = 1 ), cluster_total AS ( - SELECT cluster_id, SUM(COALESCE(cost, 0)) AS cost - FROM bill WHERE product = 'ALL_PURPOSE' AND cluster_id IS NOT NULL GROUP BY cluster_id + SELECT workspace_id, cluster_id, SUM(COALESCE(cost, 0)) AS cost + FROM bill WHERE product = 'ALL_PURPOSE' AND cluster_id IS NOT NULL GROUP BY workspace_id, cluster_id ), jobs_rate AS ( - SELECT CASE WHEN SUM(dbus) > 0 THEN SUM(COALESCE(cost, 0)) / SUM(dbus) END AS rate FROM bill WHERE product = 'JOBS' + SELECT workspace_id, CASE WHEN SUM(dbus) > 0 THEN SUM(COALESCE(cost, 0)) / SUM(dbus) END AS rate + FROM bill WHERE product = 'JOBS' GROUP BY workspace_id ), runs AS ( - SELECT r.job_id, COUNT(DISTINCT r.run_id) AS run_count, MAX(r.trigger_type) AS run_trigger + SELECT r.workspace_id, r.job_id, COUNT(DISTINCT r.run_id) AS run_count, MAX(r.trigger_type) AS run_trigger FROM system.lakeflow.job_run_timeline r CROSS JOIN p - WHERE r.workspace_id = (SELECT workspace_id FROM ws) - AND r.period_start_time >= CAST(DATE_SUB(p.we_date, 7) AS TIMESTAMP) AND r.period_start_time < CAST(p.we_date AS TIMESTAMP) - GROUP BY r.job_id + WHERE r.period_start_time >= CAST(DATE_SUB(p.we_date, 7) AS TIMESTAMP) AND r.period_start_time < CAST(p.we_date AS TIMESTAMP) + GROUP BY r.workspace_id, r.job_id ), inv AS ( - SELECT job_id, name, trigger_type FROM system.lakeflow.jobs - WHERE workspace_id = (SELECT workspace_id FROM ws) - QUALIFY ROW_NUMBER() OVER (PARTITION BY job_id ORDER BY change_time DESC) = 1 + SELECT workspace_id, job_id, name, trigger_type FROM system.lakeflow.jobs + QUALIFY ROW_NUMBER() OVER (PARTITION BY workspace_id, job_id ORDER BY change_time DESC) = 1 ) -SELECT jb.job_id, COALESCE(inv.name, 'job ' || jb.job_id) AS job_name, +SELECT jb.workspace_id, jb.job_id, COALESCE(inv.name, 'job ' || jb.job_id) AS job_name, inv.trigger_type AS trigger_type, inv.job_id IS NOT NULL AS listed_in_job_inventory, runs.run_trigger AS run_trigger_type, COALESCE(runs.run_count, 0) AS run_count, jb.job_total_cost, jb.job_all_purpose_cost, jb.job_all_purpose_dbus, CASE WHEN mc.cluster_id IS NULL OR COALESCE(ct.cost, 0) <= 0 THEN 1.0 ELSE mc.cost / ct.cost END AS job_share_of_cluster, - (SELECT rate FROM jobs_rate) AS jobs_rate_per_dbu + jr.rate AS jobs_rate_per_dbu FROM job_bill jb -LEFT JOIN runs ON runs.job_id = jb.job_id -LEFT JOIN inv ON inv.job_id = jb.job_id -LEFT JOIN main_cluster mc ON mc.job_id = jb.job_id -LEFT JOIN cluster_total ct ON ct.cluster_id = mc.cluster_id +LEFT JOIN runs ON runs.workspace_id = jb.workspace_id AND runs.job_id = jb.job_id +LEFT JOIN inv ON inv.workspace_id = jb.workspace_id AND inv.job_id = jb.job_id +LEFT JOIN main_cluster mc ON mc.workspace_id = jb.workspace_id AND mc.job_id = jb.job_id +LEFT JOIN cluster_total ct ON ct.workspace_id = mc.workspace_id AND ct.cluster_id = mc.cluster_id +LEFT JOIN jobs_rate jr ON jr.workspace_id = jb.workspace_id WHERE jb.job_all_purpose_cost > 0 ORDER BY jb.job_all_purpose_cost DESC ` @@ -319,6 +306,7 @@ export function toFacts(row: Record) { const serverless = row.serverless_price_per_dbu return { warehouse_id: id, + workspace_id: row.workspace_id == null ? null : String(row.workspace_id), warehouse_name: String(row.warehouse_name ?? id), warehouse_type: String(row.warehouse_type ?? ""), warehouse_size: row.warehouse_size == null ? null : String(row.warehouse_size), @@ -345,6 +333,7 @@ export function toClusterFacts(row: Record) { const autoTermination = row.auto_termination_minutes return { cluster_id: id, + workspace_id: row.workspace_id == null ? null : String(row.workspace_id), cluster_name: String(row.cluster_name ?? id), auto_termination_minutes: autoTermination == null ? null : Math.max(0, Math.round(num(autoTermination))), total_node_minutes: num(row.total_node_minutes), @@ -363,6 +352,7 @@ export function toJobFacts(row: Record) { const rate = row.jobs_rate_per_dbu return { job_id: id, + workspace_id: row.workspace_id == null ? null : String(row.workspace_id), job_name: String(row.job_name ?? `job ${id}`), trigger_type: row.trigger_type == null ? null : String(row.trigger_type), listed_in_job_inventory: [true, 1, "true"].includes(row.listed_in_job_inventory as never), @@ -420,7 +410,7 @@ export async function databricksSavings( try { const connector = await Registry.get(resolved.warehouse) - const binds = [window.end, runsOn] + const binds = [window.end] // A query that fills the row limit reads only its first rows; the result names it. const limited: string[] = [] const read = async (what: string, sql: string) => { @@ -477,6 +467,7 @@ export async function databricksSavings( // What each warehouse is and costs now, so a later run can tell what changed. warehouses: warehouses.map((row) => ({ id: String(row.warehouse_id ?? ""), + workspace_id: row.workspace_id == null ? null : String(row.workspace_id), name: String(row.warehouse_name ?? row.warehouse_id ?? ""), type: String(row.warehouse_type ?? ""), size: row.warehouse_size == null ? null : String(row.warehouse_size), diff --git a/packages/opencode/src/altimate/native/types.ts b/packages/opencode/src/altimate/native/types.ts index bcad388e55..201f402b7b 100644 --- a/packages/opencode/src/altimate/native/types.ts +++ b/packages/opencode/src/altimate/native/types.ts @@ -651,6 +651,8 @@ export interface DatabricksSavingsFinding { resource_type: "sql_warehouse" | "cluster" | "job" resource_id: string resource_name: string + /** The workspace the resource is in. */ + workspace_id?: string | null severity: "critical" | "high" | "medium" | "low" daily_min: number daily_max: number @@ -692,7 +694,7 @@ export interface DatabricksSavingsResult { /** What a query read only the first 10,000 of, e.g. `jobs`. */ limited?: string[] /** Each SQL warehouse as it is now, and its cost over the window. */ - warehouses?: { id: string; name: string; type: string; size: string | null; window_cost: number }[] + warehouses?: { id: string; workspace_id?: string | null; name: string; type: string; size: string | null; window_cost: number }[] report?: DatabricksSavingsReport error?: string } diff --git a/packages/opencode/src/altimate/tools/finops-databricks-savings.ts b/packages/opencode/src/altimate/tools/finops-databricks-savings.ts index 9eb9db88c6..5220b2b0d0 100644 --- a/packages/opencode/src/altimate/tools/finops-databricks-savings.ts +++ b/packages/opencode/src/altimate/tools/finops-databricks-savings.ts @@ -15,7 +15,7 @@ type SavingsMetadata = { jobs_checked?: number skipped?: { check: string; reason: string }[] limited?: string[] - warehouses?: { id: string; name: string; type: string; size: string | null; window_cost: number }[] + warehouses?: { id: string; workspace_id?: string | null; name: string; type: string; size: string | null; window_cost: number }[] report?: DatabricksSavingsReport error?: string } @@ -91,7 +91,7 @@ export function formatSavings(result: DatabricksSavingsResult): string { ) for (const finding of report.findings) { lines.push( - `${finding.resource_name} (${finding.resource_type.replace("_", " ")}) · ${finding.title} · ${range(finding.monthly_min, finding.monthly_max)} a month (${finding.severity})`, + `${finding.resource_name} (${finding.resource_type.replace("_", " ")}${finding.workspace_id ? `, workspace ${finding.workspace_id}` : ""}) · ${finding.title} · ${range(finding.monthly_min, finding.monthly_max)} a month (${finding.severity})`, ) for (const line of evidenceLines(finding)) lines.push(` ${line}`) } @@ -107,7 +107,7 @@ export function formatSavings(result: DatabricksSavingsResult): string { export const FinopsDatabricksSavingsTool = Tool.define("finops_databricks_savings", { description: - "Check a Databricks workspace for savings with Altimate's rules: SQL warehouses that would cost less as serverless or smaller, idle all-purpose clusters, and scheduled jobs running on all-purpose compute. Read-only queries over seven days of system tables (billing, compute, query history, jobs) on the connection's SQL warehouse, which Databricks starts if it is stopped. Returns each finding with its monthly dollar range and evidence, and what each flagged warehouse would cost at other sizes.", + "Check a Databricks workspace for savings with Altimate's rules: SQL warehouses that would cost less as serverless or smaller, idle all-purpose clusters, and scheduled jobs running on all-purpose compute. Read-only queries over seven days of system tables (billing, compute, query history, jobs), covering every workspace those tables show, on the connection's SQL warehouse, which Databricks starts if it is stopped. Returns each finding with its monthly dollar range and evidence, and what each flagged warehouse would cost at other sizes.", parameters: z.object({ warehouse: z .string() diff --git a/packages/opencode/test/altimate/finops-databricks-savings.test.ts b/packages/opencode/test/altimate/finops-databricks-savings.test.ts index f7339989f7..577ab5fcf5 100644 --- a/packages/opencode/test/altimate/finops-databricks-savings.test.ts +++ b/packages/opencode/test/altimate/finops-databricks-savings.test.ts @@ -154,13 +154,14 @@ describe("databricks savings handler", () => { expect(savingsWindow(Date.parse("2026-11-12T05:59:00Z"))).toEqual({ start: "2026-11-04", end: "2026-11-11" }) }) - test("the facts query is one read-only statement taking the window end and the warehouse id, in that order", () => { + test("the facts query is one read-only statement taking only the window end, over every workspace", () => { expect(DATABRICKS_WAREHOUSE_FACTS_SQL.trim()).toMatch(/^WITH\b/) expect(DATABRICKS_WAREHOUSE_FACTS_SQL).not.toMatch(/;\s*\S/) const markers = [...DATABRICKS_WAREHOUSE_FACTS_SQL.matchAll(/\?/g)].map((m) => m.index!) - expect(markers.length).toBe(2) + expect(markers.length).toBe(1) expect(DATABRICKS_WAREHOUSE_FACTS_SQL.slice(0, markers[0]!)).toContain("CAST(") - expect(DATABRICKS_WAREHOUSE_FACTS_SQL.slice(markers[0]!, markers[1]!)).toContain("WHERE workspace_id = (SELECT MAX(workspace_id)") + expect(DATABRICKS_WAREHOUSE_FACTS_SQL).not.toContain("workspace_id = (SELECT") + expect(DATABRICKS_WAREHOUSE_FACTS_SQL).toContain("SELECT inv.workspace_id, inv.warehouse_id") }) test("database values are coerced to the numbers core reads; no serverless price stays absent", () => { @@ -241,15 +242,21 @@ describe("cluster and job findings", () => { expect(result.output).toContain("Auto-termination off; $700.00 billed outside job runs over the window") }) - test("the cluster and job queries each take the window end and the warehouse id, in that order", () => { + test("the cluster and job queries each take only the window end, and key every resource by its workspace", () => { for (const sql of [DATABRICKS_CLUSTER_FACTS_SQL, DATABRICKS_JOB_FACTS_SQL]) { expect(sql.trim()).toMatch(/^WITH\b/) expect(sql).not.toMatch(/;\s*\S/) const markers = [...sql.matchAll(/\?/g)].map((m) => m.index!) - expect(markers.length).toBe(2) + expect(markers.length).toBe(1) expect(sql.slice(0, markers[0]!)).toContain("CAST(") - expect(sql.slice(markers[0]!, markers[1]!)).toContain("SELECT MAX(workspace_id) FROM system.compute.warehouses WHERE warehouse_id =") + expect(sql).not.toContain("workspace_id = (SELECT") } + expect(DATABRICKS_CLUSTER_FACTS_SQL).toContain("PARTITION BY workspace_id, cluster_id") + // Job ids repeat across workspaces: every job key pairs the workspace with the id. + expect(DATABRICKS_JOB_FACTS_SQL).toContain("PARTITION BY workspace_id, job_id") + expect(DATABRICKS_JOB_FACTS_SQL).toContain("GROUP BY workspace_id, job_id") + expect(DATABRICKS_JOB_FACTS_SQL).toContain("LEFT JOIN inv ON inv.workspace_id = jb.workspace_id AND inv.job_id = jb.job_id") + expect(DATABRICKS_JOB_FACTS_SQL).toContain("LEFT JOIN jobs_rate jr ON jr.workspace_id = jb.workspace_id") }) test("cluster and job rows are coerced to the facts core reads", () => { @@ -272,8 +279,8 @@ describe("cluster and job findings", () => { }) describe("databricks savings handler over a stubbed connection", () => { - const WAREHOUSE_ROW = { warehouse_id: "abc123", warehouse_name: "BI", warehouse_type: "PRO", warehouse_size: "SMALL", total_cost: 27.5 } - const JOB_ROW = { job_id: "j1", job_name: "nightly", trigger_type: null, listed_in_job_inventory: true, run_trigger_type: "ONE_TIME", run_count: 9 } + const WAREHOUSE_ROW = { workspace_id: "111", warehouse_id: "abc123", warehouse_name: "BI", warehouse_type: "PRO", warehouse_size: "SMALL", total_cost: 27.5 } + const JOB_ROW = { workspace_id: "222", job_id: "j1", job_name: "nightly", trigger_type: null, listed_in_job_inventory: true, run_trigger_type: "ONE_TIME", run_count: 9 } const table = (row: Record, truncated = false) => ({ columns: Object.keys(row), rows: [Object.values(row)], truncated }) let calls: { sql: string; limit: number; binds: unknown[] }[] let sent: any @@ -312,13 +319,14 @@ describe("databricks savings handler over a stubbed connection", () => { expect(result.success).toBe(true) expect(result.skipped).toEqual([{ check: "Idle Cluster Detection", reason: "[INSUFFICIENT_PERMISSIONS] system.compute.node_timeline" }]) expect([result.warehouses_checked, result.clusters_checked, result.jobs_checked]).toEqual([1, 0, 1]) - // Every query takes the window end, then the warehouse the connection runs on. + // Every query takes only the window end: it reads every workspace the system tables show. expect(calls.map((one) => one.sql)).toEqual([DATABRICKS_WAREHOUSE_FACTS_SQL, DATABRICKS_CLUSTER_FACTS_SQL, DATABRICKS_JOB_FACTS_SQL]) - for (const one of calls) expect(one.binds).toEqual(["2026-11-18", "abc123"]) + for (const one of calls) expect(one.binds).toEqual(["2026-11-18"]) expect(sent.lookback_days).toBe(7) - expect(sent.warehouses[0]).toMatchObject({ warehouse_id: "abc123", warehouse_size: "SMALL", total_cost: 27.5 }) + expect(sent.warehouses[0]).toMatchObject({ workspace_id: "111", warehouse_id: "abc123", warehouse_size: "SMALL", total_cost: 27.5 }) + expect(result.warehouses?.[0]).toMatchObject({ id: "abc123", workspace_id: "111" }) expect(sent.clusters).toEqual([]) - expect(sent.jobs[0]).toMatchObject({ job_id: "j1", trigger_type: null, listed_in_job_inventory: true, run_trigger_type: "ONE_TIME" }) + expect(sent.jobs[0]).toMatchObject({ workspace_id: "222", job_id: "j1", trigger_type: null, listed_in_job_inventory: true, run_trigger_type: "ONE_TIME" }) expect(result.limited).toEqual([]) }) @@ -362,3 +370,30 @@ describe("databricks savings handler over a stubbed connection", () => { expect(result.warehouse).toBe("dbx") }) }) + +describe("every workspace", () => { + test("a finding names its workspace in the output", () => { + const report = JSON.parse(JSON.stringify(REPORT)) + report.report.findings[0].workspace_id = "2056943136844690" + expect(formatSavings(report)).toContain("(sql warehouse, workspace 2056943136844690) · ") + }) + + test("a connection through the older /sql/1.0/endpoints/ path runs the check", async () => { + Registry.setConfigs({ + dbx: { type: "databricks", server_hostname: "dbc-0.cloud.databricks.com", http_path: "/sql/1.0/endpoints/abc123", access_token: "x" }, + }) + const spy = spyOn(Registry, "get").mockResolvedValue({ + execute: async () => ({ columns: ["warehouse_id"], rows: [["abc123"]], truncated: false }), + } as any) + const result = await databricksSavings({ warehouse: "dbx", now_ms: Date.parse("2026-11-18T10:00:00Z") }, () => ({ + lookback_days: 7, + findings: [], + total: { monthly_min: 0, monthly_max: 0, resources: 0 }, + what_if: [], + })) + spy.mockRestore() + Registry.reset() + expect(result.success).toBe(true) + expect(result.ran_on).toBe("abc123") + }) +}) From 3052130fdd583f1d49abe941e8eb111ad6f7bdde Mon Sep 17 00:00:00 2001 From: Shreyas Telkar Date: Tue, 17 Nov 2026 18:29:22 -0800 Subject: [PATCH 06/34] docs: savings tool header, refusal text and example match every-workspace scope Co-Authored-By: Claude Opus 5.5 --- docs/docs/data-engineering/tools/finops-tools.md | 4 ++-- .../src/altimate/native/finops/databricks-savings.ts | 10 +++++----- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/docs/docs/data-engineering/tools/finops-tools.md b/docs/docs/data-engineering/tools/finops-tools.md index 0212976744..03b6d5c36c 100644 --- a/docs/docs/data-engineering/tools/finops-tools.md +++ b/docs/docs/data-engineering/tools/finops-tools.md @@ -192,10 +192,10 @@ The tool measures each resource's usage; altimate-core decides what qualifies an ``` > finops_databricks_savings --warehouse prod_dbx -Databricks savings: 5 SQL warehouses, 3 all-purpose clusters, 12 jobs checked, the 7 days to 2026-11-17, at list price. +Databricks savings: 7 SQL warehouses, 3 all-purpose clusters, 12 jobs checked, the 7 days to 2026-11-17, at list price. Total: $85.17–$106 a month on 1 resource, each counted once at its largest saving. -Classic (sql warehouse) · Migrate Classic/Pro to Serverless · $85.17–$106 a month (critical) +Classic (sql warehouse, workspace 525779153077873) · Migrate Classic/Pro to Serverless · $85.17–$106 a month (critical) CLASSIC SMALL warehouse, 95% idle over 10.8 running hours (2,988 queries) $29.29 in DBUs over the window; serverless would bill $4.44 for the busy time alone ``` diff --git a/packages/opencode/src/altimate/native/finops/databricks-savings.ts b/packages/opencode/src/altimate/native/finops/databricks-savings.ts index 6ccb13e33e..44dd64de3b 100644 --- a/packages/opencode/src/altimate/native/finops/databricks-savings.ts +++ b/packages/opencode/src/altimate/native/finops/databricks-savings.ts @@ -1,9 +1,9 @@ // altimate_change - new file /** - * Databricks SQL warehouse savings: measures one row of usage facts per SQL warehouse of - * the connection's workspace from Databricks system tables, and hands them to - * altimate-core, which decides which warehouses qualify and what that saves. The rules - * and prices live in core; this file only measures. + * Databricks savings: measures usage facts per SQL warehouse, all-purpose cluster and job of + * every workspace the connection's system tables show, and hands them to altimate-core, which + * decides what qualifies and what that saves. The rules and prices live in core; this file only + * measures. */ import * as core from "@altimateai/altimate-core" @@ -400,7 +400,7 @@ export async function databricksSavings( if (!runsOn) { return failed( `Connection ${JSON.stringify(resolved.warehouse)} does not go through a SQL warehouse. ` + - "Its http_path must name one (/sql/1.0/warehouses/): the check reads that warehouse's workspace.", + "Its http_path must name one (/sql/1.0/warehouses/): the check's queries run on that warehouse.", ) } From c253ac76e66ebe327bdd5ca03fdb6ea5aeb431d2 Mon Sep 17 00:00:00 2001 From: Shreyas Telkar Date: Tue, 17 Nov 2026 18:32:06 -0800 Subject: [PATCH 07/34] test: the live savings check must succeed; docs: only cluster and job checks can be skipped Co-Authored-By: Claude Opus 5.5 --- docs/docs/data-engineering/tools/finops-tools.md | 2 +- .../test/altimate/finops-databricks-e2e.test.ts | 16 +++++++++------- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/docs/docs/data-engineering/tools/finops-tools.md b/docs/docs/data-engineering/tools/finops-tools.md index 03b6d5c36c..546ba45a4a 100644 --- a/docs/docs/data-engineering/tools/finops-tools.md +++ b/docs/docs/data-engineering/tools/finops-tools.md @@ -187,7 +187,7 @@ Check a Databricks workspace for savings with Altimate's rules, on a Databricks - **Idle Cluster Detection:** an all-purpose cluster with idle workers and auto-termination off or an hour or more. - **Job Running on All-Purpose Compute:** a scheduled job that would bill at the jobs-compute rate on a job cluster. -The tool measures each resource's usage; altimate-core decides what qualifies and what it saves. Prices are list prices. Each resource is counted once in the total, and each flagged warehouse also gets what it would cost at every smaller size and as serverless. A check whose tables the connection cannot read is named as not checked; the others still report. +The tool measures each resource's usage; altimate-core decides what qualifies and what it saves. Prices are list prices. Each resource is counted once in the total, and each flagged warehouse also gets what it would cost at every smaller size and as serverless. If the connection cannot read the cluster or job tables, that check is named as not checked and the warehouse checks still report; the warehouse tables themselves are required. ``` > finops_databricks_savings --warehouse prod_dbx diff --git a/packages/opencode/test/altimate/finops-databricks-e2e.test.ts b/packages/opencode/test/altimate/finops-databricks-e2e.test.ts index 417cf7a70a..1584efc472 100644 --- a/packages/opencode/test/altimate/finops-databricks-e2e.test.ts +++ b/packages/opencode/test/altimate/finops-databricks-e2e.test.ts @@ -209,14 +209,16 @@ describe.skipIf(!HAS_DATABRICKS)("Databricks FinOps Integration", () => { // Databricks SQL warehouse savings (facts measured here, rules in altimate-core) // ------------------------------------------------------------------------- describe("databricks_savings", () => { - test("measures every warehouse of the workspace and returns core's report", async () => { + test("measures every warehouse the system tables show and returns core's report", async () => { const result = await databricksSavings({ warehouse: WH }) - expectValidResult(result) - if (result.success) { - expect(result.warehouses_checked).toBeGreaterThan(0) - expect(Array.isArray(result.report!.findings)).toBe(true) - expect(result.report!.total.resources).toBe(new Set(result.report!.findings.map((f) => `${f.resource_type}:${f.resource_id}`)).size) - } + // A core without the rules, or a refused warehouse query, is a failure here, not a pass. + expect(result.error).toBeUndefined() + expect(result.success).toBe(true) + expect(result.warehouses_checked).toBeGreaterThan(0) + expect(Array.isArray(result.report!.findings)).toBe(true) + expect(result.report!.total.resources).toBe( + new Set(result.report!.findings.map((f) => `${f.resource_type}:${f.workspace_id ?? ""}:${f.resource_id}`)).size, + ) }, 300000) }) }) From c4511d23c6b881f8a91d3f8e966597de2e389833 Mon Sep 17 00:00:00 2001 From: Shreyas Telkar Date: Tue, 17 Nov 2026 18:58:54 -0800 Subject: [PATCH 08/34] fix: assessment mode: the packaged CLI contacts only the customer's warehouse `ALTIMATE_TELEMETRY_DISABLED=true` alone left three Altimate-side contacts on the packaged `tool run` path: the one-time `telemetry_opted_out` send to the telemetry endpoint, the models.dev catalog refresh, and the ripgrep download from github.com. `ALTIMATE_ASSESSMENT_MODE=true` now skips all three: telemetry init returns before resolving a sink or touching the opt-out marker and machine id, the models fetch flag turns on, and the ripgrep download refuses. A packaged-binary test runs the built CLI behind a logging proxy that refuses every connection and fails on any host: startup (`tool list`), failure (no warehouse, unreachable warehouse), opt-out by env and by project config, telemetry left on, and a control run without the mode that must reach the telemetry endpoint. It also checks the marker and machine id are untouched. Removing any one of the three guards fails it. Docs state the window, what is not assessed, and the mode. Co-Authored-By: Claude Sonnet 5.5 --- .github/workflows/ci.yml | 1 + .../data-engineering/tools/finops-tools.md | 4 + packages/core/src/flag/flag.ts | 5 +- packages/core/src/ripgrep/binary.ts | 4 + .../opencode/src/altimate/telemetry/index.ts | 7 + packages/opencode/src/file/ripgrep.ts | 4 + packages/opencode/src/flag/flag.ts | 5 +- .../altimate/assessment-zero-egress.test.ts | 166 ++++++++++++++++++ 8 files changed, 194 insertions(+), 2 deletions(-) create mode 100644 packages/opencode/test/altimate/assessment-zero-egress.test.ts diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e9a7d7150d..b4c9aed7ab 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -26,6 +26,7 @@ env: PROCESS_SUITES: >- test/altimate/learn/bootstrap-cli.test.ts test/altimate/tool-cli/cli.test.ts + test/altimate/assessment-zero-egress.test.ts test/altimate/rules/host-cli.test.ts test/upstream/bridge-merge-e2e.test.ts diff --git a/docs/docs/data-engineering/tools/finops-tools.md b/docs/docs/data-engineering/tools/finops-tools.md index 546ba45a4a..d041f7da30 100644 --- a/docs/docs/data-engineering/tools/finops-tools.md +++ b/docs/docs/data-engineering/tools/finops-tools.md @@ -189,6 +189,10 @@ Check a Databricks workspace for savings with Altimate's rules, on a Databricks The tool measures each resource's usage; altimate-core decides what qualifies and what it saves. Prices are list prices. Each resource is counted once in the total, and each flagged warehouse also gets what it would cost at every smaller size and as serverless. If the connection cannot read the cluster or job tables, that check is named as not checked and the warehouse checks still report; the warehouse tables themselves are required. +**What it does not assess.** Serverless warehouses and Databricks Apps spend are not covered. The cost of each resource is the billed usage of the six full days before the window's last day, scaled to seven; on one measured workspace that read about 7% below the seven billed days. The connection needs the `@databricks/sql` driver installed. + +**Assessment mode.** Set `ALTIMATE_ASSESSMENT_MODE=true` to keep the run from contacting anything but your warehouse. It sends no telemetry (not even the one-time opt-out event), and downloads neither the model catalog (models.dev) nor ripgrep (github.com). Without it, `ALTIMATE_TELEMETRY_DISABLED=true` alone still lets the CLI send that opt-out event and fetch those. The Claude Code `/savings` command sets it for you. Tool results shown in a Claude Code conversation reach its model provider; assessment mode does not change that. + ``` > finops_databricks_savings --warehouse prod_dbx diff --git a/packages/core/src/flag/flag.ts b/packages/core/src/flag/flag.ts index d85627f1b5..8de4d153ef 100644 --- a/packages/core/src/flag/flag.ts +++ b/packages/core/src/flag/flag.ts @@ -97,7 +97,10 @@ export const Flag = { OPENCODE_DISABLE_TERMINAL_TITLE: truthy("OPENCODE_DISABLE_TERMINAL_TITLE"), OPENCODE_SHOW_TTFD: truthy("OPENCODE_SHOW_TTFD"), OPENCODE_DISABLE_AUTOCOMPACT: truthy("OPENCODE_DISABLE_AUTOCOMPACT"), - OPENCODE_DISABLE_MODELS_FETCH: truthy("OPENCODE_DISABLE_MODELS_FETCH"), + // altimate_change start — assessment mode: nothing leaves the machine except the customer's own warehouse calls + ALTIMATE_ASSESSMENT_MODE: truthy("ALTIMATE_ASSESSMENT_MODE"), + OPENCODE_DISABLE_MODELS_FETCH: truthy("OPENCODE_DISABLE_MODELS_FETCH") || truthy("ALTIMATE_ASSESSMENT_MODE"), + // altimate_change end OPENCODE_DISABLE_MOUSE: truthy("OPENCODE_DISABLE_MOUSE"), // altimate_change start — documented ALTIMATE_CLI_ name read first (see `env`) OPENCODE_FAKE_VCS: env("OPENCODE_FAKE_VCS"), diff --git a/packages/core/src/ripgrep/binary.ts b/packages/core/src/ripgrep/binary.ts index 3f482ccb87..c593c18cef 100644 --- a/packages/core/src/ripgrep/binary.ts +++ b/packages/core/src/ripgrep/binary.ts @@ -8,6 +8,7 @@ import { LayerNode } from "../effect/layer-node" import { httpClient } from "../effect/layer-node-platform" import { FSUtil } from "../fs-util" import { Global } from "../global" +import { Flag } from "../flag/flag" import { which } from "../util/which" // altimate_change start — upstream_fix: unzip in-process instead of shelling out to PowerShell. import { randomUUID } from "node:crypto" @@ -173,6 +174,9 @@ export namespace RipgrepBinary { const config = PLATFORM[platformKey] if (!config) throw new Error(`unsupported platform for ripgrep: ${platformKey}`) + // altimate_change start — assessment mode downloads nothing: a missing ripgrep fails the search + if (Flag.ALTIMATE_ASSESSMENT_MODE) throw new Error("ripgrep is not installed and downloads are off in assessment mode") + // altimate_change end const filename = `ripgrep-${VERSION}-${config.platform}.${config.extension}` const url = `https://github.com/BurntSushi/ripgrep/releases/download/${VERSION}/${filename}` diff --git a/packages/opencode/src/altimate/telemetry/index.ts b/packages/opencode/src/altimate/telemetry/index.ts index 6ecb4cfead..cb39650041 100644 --- a/packages/opencode/src/altimate/telemetry/index.ts +++ b/packages/opencode/src/altimate/telemetry/index.ts @@ -2510,6 +2510,13 @@ export namespace Telemetry { async function doInit() { try { + // altimate_change start — assessment mode sends nothing to Altimate, not even the one-time + // opt-out event: no sink is resolved and no marker or machine id is touched. + if (Flag.truthyEnv("ALTIMATE_ASSESSMENT_MODE")) { + buffer = [] + return + } + // altimate_change end // altimate_change — accept "true"/"TRUE"/"1" (case-insensitive) via truthyEnv, // and honor the OPENCODE_DISABLE_TELEMETRY fallback promised by v0.9.4's CHANGELOG // (previously only wired in test fixtures, silent no-op in product). diff --git a/packages/opencode/src/file/ripgrep.ts b/packages/opencode/src/file/ripgrep.ts index d2356219d6..7e4fa5f4ed 100644 --- a/packages/opencode/src/file/ripgrep.ts +++ b/packages/opencode/src/file/ripgrep.ts @@ -8,6 +8,7 @@ // Ripgrep utility functions import path from "path" import { Global } from "../global" +import { Flag } from "../flag/flag" import fs from "fs/promises" import z from "zod" import { NamedError } from "@opencode-ai/util/error" @@ -100,6 +101,9 @@ export namespace Ripgrep { const config = PLATFORM[platformKey] if (!config) throw new UnsupportedPlatformError({ platform: platformKey }) + // altimate_change start — assessment mode downloads nothing: a missing ripgrep fails the search + if (Flag.ALTIMATE_ASSESSMENT_MODE) throw new Error("ripgrep is not installed and downloads are off in assessment mode") + // altimate_change end const version = "14.1.1" const filename = `ripgrep-${version}-${config.platform}.${config.extension}` const url = `https://github.com/BurntSushi/ripgrep/releases/download/${version}/${filename}` diff --git a/packages/opencode/src/flag/flag.ts b/packages/opencode/src/flag/flag.ts index a833dedff5..0fdb299cf1 100644 --- a/packages/opencode/src/flag/flag.ts +++ b/packages/opencode/src/flag/flag.ts @@ -113,7 +113,10 @@ export namespace Flag { export const OPENCODE_DISABLE_LSP_DOWNLOAD = truthy("OPENCODE_DISABLE_LSP_DOWNLOAD") export const OPENCODE_ENABLE_EXPERIMENTAL_MODELS = truthy("OPENCODE_ENABLE_EXPERIMENTAL_MODELS") export const OPENCODE_DISABLE_AUTOCOMPACT = truthy("OPENCODE_DISABLE_AUTOCOMPACT") - export const OPENCODE_DISABLE_MODELS_FETCH = truthy("OPENCODE_DISABLE_MODELS_FETCH") + // altimate_change start — assessment mode: nothing leaves the machine except the customer's own warehouse calls + export const ALTIMATE_ASSESSMENT_MODE = truthy("ALTIMATE_ASSESSMENT_MODE") + export const OPENCODE_DISABLE_MODELS_FETCH = truthy("OPENCODE_DISABLE_MODELS_FETCH") || ALTIMATE_ASSESSMENT_MODE + // altimate_change end export const OPENCODE_DISABLE_CLAUDE_CODE = truthy("OPENCODE_DISABLE_CLAUDE_CODE") export const OPENCODE_DISABLE_CLAUDE_CODE_PROMPT = OPENCODE_DISABLE_CLAUDE_CODE || truthy("OPENCODE_DISABLE_CLAUDE_CODE_PROMPT") diff --git a/packages/opencode/test/altimate/assessment-zero-egress.test.ts b/packages/opencode/test/altimate/assessment-zero-egress.test.ts new file mode 100644 index 0000000000..a3cec8eb9f --- /dev/null +++ b/packages/opencode/test/altimate/assessment-zero-egress.test.ts @@ -0,0 +1,166 @@ +// The packaged CLI, run the way the Claude Code plugin's /savings runs it, must not contact any host +// but the customer's warehouse. A logging proxy refuses every connection and records its target; the +// control run (assessment mode off) proves the proxy would have seen Altimate's telemetry endpoint. +import { afterAll, beforeAll, describe, expect, test } from "bun:test" +import http from "node:http" +import type { AddressInfo } from "node:net" +import { mkdir, mkdtemp, readFile, readdir, rm, stat, writeFile } from "node:fs/promises" +import { tmpdir } from "node:os" +import path from "node:path" + +const root = path.join(import.meta.dir, "../..") +const snapshot = path.join(root, "src/provider/models-snapshot.ts") +const posixOnly = test.skipIf(process.platform === "win32") +// Stands in for Altimate's telemetry endpoint, so a send to it is attributable and cannot race the +// real one's availability. +const TELEMETRY_SINK = "altimate-telemetry.invalid" +const MACHINE_ID = "11111111-2222-4333-8444-555555555555" + +let binary = "" +let originalSnapshot: string | undefined +let proxy: http.Server +let proxyPort = 0 +let seen: string[] = [] +const scratch: string[] = [] + +/** The ambient environment without anything that makes the CLI mistake this for a test run (which silences + * its telemetry for its own reasons) or that redirects its settings. The build must not see it either: + * the bundle can capture `NODE_ENV`. */ +function cleanEnv(): Record { + const base: Record = {} + for (const [key, value] of Object.entries(process.env)) { + if (value !== undefined && !/^(NODE_ENV|BUN_TEST|VITEST|JEST_WORKER_ID|ALTIMATE_TELEMETRY_.*|ALTIMATE_ASSESSMENT_MODE|OPENCODE_.*|APPLICATIONINSIGHTS_.*)$/.test(key)) base[key] = value + } + return base +} + +async function packagedBinary(): Promise { + const given = process.env.ALTIMATE_PACKAGED_BIN + if (given) return given + // The build rewrites the tracked models snapshot; put it back afterwards. + originalSnapshot = await readFile(snapshot, "utf8") + const build = Bun.spawnSync([process.execPath, "run", "script/build.ts", "--single", "--skip-install"], { + cwd: root, + stdout: "pipe", + stderr: "pipe", + env: cleanEnv(), + }) + if (build.exitCode !== 0) throw new Error(`build failed: ${build.stderr.toString()}`) + const dist = path.join(root, "dist") + for (const pkg of await readdir(path.join(dist, "@altimateai"))) { + const bin = path.join(dist, "@altimateai", pkg, "bin", "altimate-code") + if (await stat(bin).then(() => true, () => false)) return bin + } + throw new Error("no packaged binary was built") +} + +beforeAll(async () => { + if (process.platform === "win32") return + binary = await packagedBinary() + proxy = http.createServer((req, res) => { + seen.push(new URL(req.url ?? "/", "http://x").host || String(req.headers.host)) + res.writeHead(502).end() + }) + proxy.on("connect", (req, socket) => { + seen.push(String(req.url)) + socket.end("HTTP/1.1 502 Bad Gateway\r\n\r\n") + }) + await new Promise((resolve) => proxy.listen(0, "127.0.0.1", resolve)) + proxyPort = (proxy.address() as AddressInfo).port +}, 120_000) + +afterAll(async () => { + proxy?.close() + if (originalSnapshot !== undefined) await writeFile(snapshot, originalSnapshot) + await Promise.all(scratch.map((dir) => rm(dir, { recursive: true, force: true }))) +}) + +type Run = { code: number; stdout: string; hosts: string[]; home: string } + +/** Runs the packaged CLI in a fresh home that already holds a machine id (so an opt-out event could be sent). */ +async function run(args: string[], env: Record, setup?: (home: string, project: string) => Promise): Promise { + const home = await mkdtemp(path.join(tmpdir(), "assessment-egress-")) + scratch.push(home) + const project = path.join(home, "project") + await mkdir(path.join(home, ".altimate"), { recursive: true }) + await mkdir(project, { recursive: true }) + await writeFile(path.join(home, ".altimate", "machine-id"), `${MACHINE_ID}\n`) + await setup?.(home, project) + seen = [] + const base = cleanEnv() + const proc = Bun.spawn([binary, ...args], { + cwd: project, + stdout: "pipe", + stderr: "pipe", + stdin: "ignore", + timeout: 90_000, + env: { + ...base, + HOME: home, + XDG_CONFIG_HOME: path.join(home, ".config"), + XDG_DATA_HOME: path.join(home, ".local/share"), + XDG_CACHE_HOME: path.join(home, ".cache"), + XDG_STATE_HOME: path.join(home, ".local/state"), + ALTIMATE_TOOL_DAEMON: "0", + HTTPS_PROXY: `http://127.0.0.1:${proxyPort}`, + HTTP_PROXY: `http://127.0.0.1:${proxyPort}`, + NO_PROXY: "", + APPLICATIONINSIGHTS_CONNECTION_STRING: `InstrumentationKey=00000000-0000-0000-0000-000000000000;IngestionEndpoint=https://${TELEMETRY_SINK}/`, + ...env, + }, + }) + const [stdout, code] = await Promise.all([new Response(proc.stdout).text(), proc.exited]) + return { code, stdout, hosts: [...new Set(seen)].sort(), home } +} + +const assessment = { ALTIMATE_ASSESSMENT_MODE: "true", ALTIMATE_TELEMETRY_DISABLED: "true" } +const savings = ["tool", "run", "finops_databricks_savings", "--args", "{}", "--json", "--no-daemon"] + +describe("assessment mode on the packaged CLI", () => { + posixOnly("control: without assessment mode, telemetry off still sends the opt-out event to the telemetry endpoint", async () => { + const result = await run(savings, { ALTIMATE_TELEMETRY_DISABLED: "true" }) + expect(result.hosts.some((host) => host.startsWith(TELEMETRY_SINK))).toBe(true) + }, 120_000) + + posixOnly("failure path (no warehouse configured): no host is contacted and the run reports the failure", async () => { + const result = await run(savings, assessment) + expect(result.hosts).toEqual([]) + expect(JSON.parse(result.stdout).status).toBe("error") + }, 120_000) + + posixOnly("startup path (`tool list`): no host is contacted", async () => { + const result = await run(["tool", "list"], assessment) + expect(result.code).toBe(0) + expect(result.hosts).toEqual([]) + }, 120_000) + + posixOnly("opt-out by project config: no host is contacted", async () => { + const result = await run(savings, { ALTIMATE_ASSESSMENT_MODE: "true" }, async (_home, project) => { + await writeFile(path.join(project, "altimate-code.json"), JSON.stringify({ telemetry: { disabled: true } })) + }) + expect(result.hosts).toEqual([]) + }, 120_000) + + posixOnly("telemetry left on: no host is contacted either", async () => { + const result = await run(savings, { ALTIMATE_ASSESSMENT_MODE: "true" }) + expect(result.hosts).toEqual([]) + }, 120_000) + + posixOnly("an unreachable warehouse still contacts no other host", async () => { + const result = await run(savings, assessment, async (_home, project) => { + const connections = path.join(project, "connections.json") + await writeFile( + connections, + JSON.stringify({ dbx: { type: "databricks", server_hostname: "127.0.0.1", http_path: "/sql/1.0/warehouses/x", access_token: "unused" } }), + ) + }) + expect(result.hosts.filter((host) => !host.startsWith("127.0.0.1"))).toEqual([]) + }, 120_000) + + posixOnly("the opt-out marker and machine id are untouched", async () => { + const result = await run(savings, assessment) + const altimate = await readdir(path.join(result.home, ".altimate")) + expect(altimate.sort()).toEqual(["machine-id"]) + expect((await readFile(path.join(result.home, ".altimate", "machine-id"), "utf8")).trim()).toBe(MACHINE_ID) + }, 120_000) +}) From 19d5cddfa5b6279b48bcd10c3814d92873007bd8 Mon Sep 17 00:00:00 2001 From: Shreyas Telkar Date: Tue, 17 Nov 2026 19:20:16 -0800 Subject: [PATCH 09/34] docs: state the savings cost window as the queries and core compute it A SQL warehouse's cost is the six full days before the window's last day, counting only billing ingested before that day, spread over the seven-day window as the pipeline does; it was described as "scaled to seven", which nothing does. Clusters and jobs read the seven full days with no ingestion cutoff. Co-Authored-By: Claude Opus 5.5 --- docs/docs/data-engineering/tools/finops-tools.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/docs/data-engineering/tools/finops-tools.md b/docs/docs/data-engineering/tools/finops-tools.md index d041f7da30..43e8b96350 100644 --- a/docs/docs/data-engineering/tools/finops-tools.md +++ b/docs/docs/data-engineering/tools/finops-tools.md @@ -189,7 +189,7 @@ Check a Databricks workspace for savings with Altimate's rules, on a Databricks The tool measures each resource's usage; altimate-core decides what qualifies and what it saves. Prices are list prices. Each resource is counted once in the total, and each flagged warehouse also gets what it would cost at every smaller size and as serverless. If the connection cannot read the cluster or job tables, that check is named as not checked and the warehouse checks still report; the warehouse tables themselves are required. -**What it does not assess.** Serverless warehouses and Databricks Apps spend are not covered. The cost of each resource is the billed usage of the six full days before the window's last day, scaled to seven; on one measured workspace that read about 7% below the seven billed days. The connection needs the `@databricks/sql` driver installed. +**What it does not assess.** Serverless warehouses and Databricks Apps spend are not covered. A SQL warehouse's cost is its billed usage for the six full days before the window's last day, counting only records Databricks had ingested before that day, spread over the seven-day window as Altimate's pipeline does, so it reads lower than seven days of billing (on one measured workspace, about 7% lower). A cluster's or job's cost is its billed usage for the seven full days before the window's last day. The connection needs the `@databricks/sql` driver installed. **Assessment mode.** Set `ALTIMATE_ASSESSMENT_MODE=true` to keep the run from contacting anything but your warehouse. It sends no telemetry (not even the one-time opt-out event), and downloads neither the model catalog (models.dev) nor ripgrep (github.com). Without it, `ALTIMATE_TELEMETRY_DISABLED=true` alone still lets the CLI send that opt-out event and fetch those. The Claude Code `/savings` command sets it for you. Tool results shown in a Claude Code conversation reach its model provider; assessment mode does not change that. From ffd8be64c3fb08b84c257fcba9f47e43f2837d56 Mon Sep 17 00:00:00 2001 From: Shreyas Telkar Date: Tue, 17 Nov 2026 20:36:44 -0800 Subject: [PATCH 10/34] fix: assessment mode also covers sign-in lookups, remote config, plugin installs and the daemon Review of the first assessment-mode commit found contacts it left open on the packaged `tool run` path: - A signed-in user's project-binding lookup went to the Altimate backend with the sign-in and absolute project path. Assessment mode now treats every caller as signed out (`isConfigured()` and the workspace request helper), turns the workspace paths off, and leaves `machine_id` out of the sign-in context. - A `wellknown` auth entry fetched remote config, and a project with plugin or tool sources ran an npm install. Both are skipped. - A tool daemon started without the mode served a run that set it; the variable is now part of the daemon's identity. - An explicit model-catalog refresh contacted models.dev; it is a no-op in the mode. - The switch now fails closed like the other kill switches: any value but empty, `0` or `false` turns it on. Tests: the packaged-binary suite runs with a scrubbed environment (the test runner's own `ALTIMATE_*` and `XDG_*` values leaked into the child and hid the sign-in case), adds signed-in, remote-config and plugin-source, `yes`, daemon-identity, core ripgrep, flag and sign-in-context cases, and a real unreachable-warehouse case. Removing each guard fails at least one test. Co-Authored-By: Claude Sonnet 5.5 --- .../data-engineering/tools/finops-tools.md | 2 +- packages/core/src/flag/flag.ts | 10 +- packages/core/src/models-dev.ts | 3 + packages/core/src/ripgrep/binary.ts | 2 +- packages/opencode/src/altimate/api/client.ts | 3 + .../opencode/src/altimate/plugin/altimate.ts | 2 +- .../opencode/src/altimate/telemetry/index.ts | 2 +- .../opencode/src/altimate/tool-cli/daemon.ts | 2 +- .../src/altimate/workspace/api-client.ts | 4 + packages/opencode/src/config/config.ts | 7 +- packages/opencode/src/file/ripgrep.ts | 2 +- packages/opencode/src/flag/flag.ts | 8 +- packages/opencode/src/provider/models.ts | 3 + .../altimate/assessment-zero-egress.test.ts | 104 ++++++++++++++++-- .../fixtures/assessment-flags-probe.ts | 12 ++ .../fixtures/assessment-misc-probe.ts | 11 ++ .../altimate/fixtures/assessment-rg-probe.ts | 6 + 17 files changed, 161 insertions(+), 22 deletions(-) create mode 100644 packages/opencode/test/altimate/fixtures/assessment-flags-probe.ts create mode 100644 packages/opencode/test/altimate/fixtures/assessment-misc-probe.ts create mode 100644 packages/opencode/test/altimate/fixtures/assessment-rg-probe.ts diff --git a/docs/docs/data-engineering/tools/finops-tools.md b/docs/docs/data-engineering/tools/finops-tools.md index 43e8b96350..4b98ca7415 100644 --- a/docs/docs/data-engineering/tools/finops-tools.md +++ b/docs/docs/data-engineering/tools/finops-tools.md @@ -191,7 +191,7 @@ The tool measures each resource's usage; altimate-core decides what qualifies an **What it does not assess.** Serverless warehouses and Databricks Apps spend are not covered. A SQL warehouse's cost is its billed usage for the six full days before the window's last day, counting only records Databricks had ingested before that day, spread over the seven-day window as Altimate's pipeline does, so it reads lower than seven days of billing (on one measured workspace, about 7% lower). A cluster's or job's cost is its billed usage for the seven full days before the window's last day. The connection needs the `@databricks/sql` driver installed. -**Assessment mode.** Set `ALTIMATE_ASSESSMENT_MODE=true` to keep the run from contacting anything but your warehouse. It sends no telemetry (not even the one-time opt-out event), and downloads neither the model catalog (models.dev) nor ripgrep (github.com). Without it, `ALTIMATE_TELEMETRY_DISABLED=true` alone still lets the CLI send that opt-out event and fetch those. The Claude Code `/savings` command sets it for you. Tool results shown in a Claude Code conversation reach its model provider; assessment mode does not change that. +**Assessment mode.** Set `ALTIMATE_ASSESSMENT_MODE=true` to keep the run from contacting anything but your warehouse. It sends no telemetry (not even the one-time opt-out event), treats you as signed out of Altimate (no workspace or project-binding lookups), fetches no remote config and installs no plugin dependencies, and downloads neither the model catalog (models.dev) nor ripgrep (github.com). Any value but empty, `0` or `false` turns it on, and a tool daemon started without it is not reused. It is measured for `tool run`; other commands (the update check, language-server downloads) are not covered. Core's compiled analysis is not audited for network use. Without it, `ALTIMATE_TELEMETRY_DISABLED=true` alone still lets the CLI send that opt-out event and fetch those. The Claude Code `/savings` command sets it for you. Tool results shown in a Claude Code conversation reach its model provider; assessment mode does not change that. ``` > finops_databricks_savings --warehouse prod_dbx diff --git a/packages/core/src/flag/flag.ts b/packages/core/src/flag/flag.ts index 8de4d153ef..624e3687a5 100644 --- a/packages/core/src/flag/flag.ts +++ b/packages/core/src/flag/flag.ts @@ -98,8 +98,11 @@ export const Flag = { OPENCODE_SHOW_TTFD: truthy("OPENCODE_SHOW_TTFD"), OPENCODE_DISABLE_AUTOCOMPACT: truthy("OPENCODE_DISABLE_AUTOCOMPACT"), // altimate_change start — assessment mode: nothing leaves the machine except the customer's own warehouse calls - ALTIMATE_ASSESSMENT_MODE: truthy("ALTIMATE_ASSESSMENT_MODE"), - OPENCODE_DISABLE_MODELS_FETCH: truthy("OPENCODE_DISABLE_MODELS_FETCH") || truthy("ALTIMATE_ASSESSMENT_MODE"), + // Fails closed like the other kill switches: `yes` or `on` must not leave the CLI contacting hosts. + get ALTIMATE_ASSESSMENT_MODE() { + return killSwitchSet(process.env["ALTIMATE_ASSESSMENT_MODE"]) + }, + OPENCODE_DISABLE_MODELS_FETCH: truthy("OPENCODE_DISABLE_MODELS_FETCH") || killSwitchSet(process.env["ALTIMATE_ASSESSMENT_MODE"]), // altimate_change end OPENCODE_DISABLE_MOUSE: truthy("OPENCODE_DISABLE_MOUSE"), // altimate_change start — documented ALTIMATE_CLI_ name read first (see `env`) @@ -134,7 +137,8 @@ export const Flag = { // runtime `--` middleware can flip it between plugin activation and command execution. // Unrelated to OPENCODE_EXPERIMENTAL_WORKSPACES (multi-instance control plane). get ALTIMATE_DISABLE_WORKSPACE() { - return killSwitchSet(env("ALTIMATE_DISABLE_WORKSPACE")) + // Assessment mode also turns the SaaS workspace paths off: they send the project path and sign-in to Altimate. + return killSwitchSet(env("ALTIMATE_DISABLE_WORKSPACE")) || killSwitchSet(process.env["ALTIMATE_ASSESSMENT_MODE"]) }, /** * Workspace precedence escape hatch, set by `--integrations=local`. When on, the diff --git a/packages/core/src/models-dev.ts b/packages/core/src/models-dev.ts index 3f9f670374..baa3aa6406 100644 --- a/packages/core/src/models-dev.ts +++ b/packages/core/src/models-dev.ts @@ -217,6 +217,9 @@ export const layer = Layer.effect( const get = (): Effect.Effect> => cachedGet const refresh = Effect.fn("ModelsDev.refresh")(function* (force = false) { + // altimate_change start — assessment mode contacts no catalog host, even when asked to refresh + if (Flag.ALTIMATE_ASSESSMENT_MODE) return + // altimate_change end if (!force && (yield* fresh())) return yield* Effect.scoped( Effect.gen(function* () { diff --git a/packages/core/src/ripgrep/binary.ts b/packages/core/src/ripgrep/binary.ts index c593c18cef..9f5256ceb5 100644 --- a/packages/core/src/ripgrep/binary.ts +++ b/packages/core/src/ripgrep/binary.ts @@ -175,7 +175,7 @@ export namespace RipgrepBinary { if (!config) throw new Error(`unsupported platform for ripgrep: ${platformKey}`) // altimate_change start — assessment mode downloads nothing: a missing ripgrep fails the search - if (Flag.ALTIMATE_ASSESSMENT_MODE) throw new Error("ripgrep is not installed and downloads are off in assessment mode") + if (Flag.ALTIMATE_ASSESSMENT_MODE) return yield* Effect.fail(new Error("ripgrep is not installed and downloads are off in assessment mode")) // altimate_change end const filename = `ripgrep-${VERSION}-${config.platform}.${config.extension}` const url = `https://github.com/BurntSushi/ripgrep/releases/download/${VERSION}/${filename}` diff --git a/packages/opencode/src/altimate/api/client.ts b/packages/opencode/src/altimate/api/client.ts index 4c1b153ca1..9a6e5278ea 100644 --- a/packages/opencode/src/altimate/api/client.ts +++ b/packages/opencode/src/altimate/api/client.ts @@ -11,6 +11,7 @@ import { readMcpEntryFromDisk, removeMcpFromConfig, } from "../../mcp/config" +import { Flag } from "../../flag/flag" import { InstallationVersion } from "@opencode-ai/core/installation/version" const DEFAULT_MCP_URL = "https://mcpserver.getaltimate.com/sse" @@ -82,6 +83,8 @@ export namespace AltimateApi { } export async function isConfigured(): Promise { + // Assessment mode talks to no Altimate service: to every caller the user is signed out. + if (Flag.assessmentMode()) return false if (currentCredentials()) return true return Filesystem.exists(credentialsPath()) } diff --git a/packages/opencode/src/altimate/plugin/altimate.ts b/packages/opencode/src/altimate/plugin/altimate.ts index baf989aa1d..5a8c632387 100644 --- a/packages/opencode/src/altimate/plugin/altimate.ts +++ b/packages/opencode/src/altimate/plugin/altimate.ts @@ -86,7 +86,7 @@ export async function buildCliContext(machineIdPath?: string, directory?: string // ("true"/"TRUE"/"1", case-insensitive — see Flag.truthyEnv) // 2. config.telemetry.disabled (resolved via the async Config.get()) let disabled = - Flag.truthyEnv("ALTIMATE_TELEMETRY_DISABLED") || Flag.truthyEnv("OPENCODE_DISABLE_TELEMETRY") + Flag.truthyEnv("ALTIMATE_TELEMETRY_DISABLED") || Flag.truthyEnv("OPENCODE_DISABLE_TELEMETRY") || Flag.assessmentMode() if (!disabled) { try { const userConfig = (await Config.get()) as any diff --git a/packages/opencode/src/altimate/telemetry/index.ts b/packages/opencode/src/altimate/telemetry/index.ts index cb39650041..2d18dbac6b 100644 --- a/packages/opencode/src/altimate/telemetry/index.ts +++ b/packages/opencode/src/altimate/telemetry/index.ts @@ -2512,7 +2512,7 @@ export namespace Telemetry { try { // altimate_change start — assessment mode sends nothing to Altimate, not even the one-time // opt-out event: no sink is resolved and no marker or machine id is touched. - if (Flag.truthyEnv("ALTIMATE_ASSESSMENT_MODE")) { + if (Flag.assessmentMode()) { buffer = [] return } diff --git a/packages/opencode/src/altimate/tool-cli/daemon.ts b/packages/opencode/src/altimate/tool-cli/daemon.ts index e28a10f510..37d405febc 100644 --- a/packages/opencode/src/altimate/tool-cli/daemon.ts +++ b/packages/opencode/src/altimate/tool-cli/daemon.ts @@ -45,7 +45,7 @@ const CONN_ENV_PREFIX = "ALTIMATE_CODE_CONN_" // Variables a daemon reads once at spawn that change what it returns or exposes (value sampling, integration // routing, workspace tools), so a caller with a different value must not be served by a daemon started without it. -const BEHAVIOUR_ENV = ["ALTIMATE_DISABLE_SCHEMA_VALUES", "ALTIMATE_INTEGRATIONS", "ALTIMATE_DISABLE_WORKSPACE"] +const BEHAVIOUR_ENV = ["ALTIMATE_DISABLE_SCHEMA_VALUES", "ALTIMATE_INTEGRATIONS", "ALTIMATE_DISABLE_WORKSPACE", "ALTIMATE_ASSESSMENT_MODE"] /** A short digest of the caller's `ALTIMATE_CODE_CONN_*` variables and the behaviour variables above, empty when * none are set. A daemon reads them once at spawn, so callers with different values must not share one. */ diff --git a/packages/opencode/src/altimate/workspace/api-client.ts b/packages/opencode/src/altimate/workspace/api-client.ts index 5a35b8c549..38145ba889 100644 --- a/packages/opencode/src/altimate/workspace/api-client.ts +++ b/packages/opencode/src/altimate/workspace/api-client.ts @@ -12,6 +12,7 @@ // for 409/412 and surface it as a typed error rather than a bare status code. import { AltimateApi } from "@/altimate/api/client" import { Telemetry } from "@/altimate/telemetry" +import { Flag } from "@/flag/flag" const REQUEST_TIMEOUT_MS = 15_000 @@ -395,6 +396,9 @@ async function req( // altimate_change end } = {}, ): Promise { + // altimate_change start — assessment mode talks to no Altimate service, whoever the caller acts as + if (Flag.assessmentMode()) throw new NotConfiguredError() + // altimate_change end const timeoutMs = opts.timeoutMs ?? REQUEST_TIMEOUT_MS const { url, instance, apiKey } = opts.actAs ?? (await creds()) const qs = opts.query ? "?" + new URLSearchParams(opts.query).toString() : "" diff --git a/packages/opencode/src/config/config.ts b/packages/opencode/src/config/config.ts index d49756dc52..b5db4577d7 100644 --- a/packages/opencode/src/config/config.ts +++ b/packages/opencode/src/config/config.ts @@ -535,6 +535,9 @@ export const layer = Layer.effect( } for (const [key, value] of Object.entries(auth)) { + // altimate_change start — assessment mode fetches no remote config + if (value.type === "wellknown" && Flag.ALTIMATE_ASSESSMENT_MODE) continue + // altimate_change end if (value.type === "wellknown") { const url = key.replace(/\/+$/, "") authEnv[value.key] = value.token @@ -744,7 +747,9 @@ export const layer = Layer.effect( // under any earlier directory. Keep the PURE skip and retain fibers for waitForDependencies. const deps: Fiber.Fiber[] = [] for (const dir of directories) { - if (ConfigPlugin.shouldInstallDependencies(dir, result.plugin)) { + // altimate_change start — assessment mode installs no plugin dependencies (npm) + if (!Flag.ALTIMATE_ASSESSMENT_MODE && ConfigPlugin.shouldInstallDependencies(dir, result.plugin)) { + // altimate_change end const dep = yield* npmSvc .install(dir, { add: [ diff --git a/packages/opencode/src/file/ripgrep.ts b/packages/opencode/src/file/ripgrep.ts index 7e4fa5f4ed..265bb7fbbf 100644 --- a/packages/opencode/src/file/ripgrep.ts +++ b/packages/opencode/src/file/ripgrep.ts @@ -102,7 +102,7 @@ export namespace Ripgrep { if (!config) throw new UnsupportedPlatformError({ platform: platformKey }) // altimate_change start — assessment mode downloads nothing: a missing ripgrep fails the search - if (Flag.ALTIMATE_ASSESSMENT_MODE) throw new Error("ripgrep is not installed and downloads are off in assessment mode") + if (Flag.assessmentMode()) throw new Error("ripgrep is not installed and downloads are off in assessment mode") // altimate_change end const version = "14.1.1" const filename = `ripgrep-${version}-${config.platform}.${config.extension}` diff --git a/packages/opencode/src/flag/flag.ts b/packages/opencode/src/flag/flag.ts index 0fdb299cf1..cb97cc06a3 100644 --- a/packages/opencode/src/flag/flag.ts +++ b/packages/opencode/src/flag/flag.ts @@ -114,8 +114,12 @@ export namespace Flag { export const OPENCODE_ENABLE_EXPERIMENTAL_MODELS = truthy("OPENCODE_ENABLE_EXPERIMENTAL_MODELS") export const OPENCODE_DISABLE_AUTOCOMPACT = truthy("OPENCODE_DISABLE_AUTOCOMPACT") // altimate_change start — assessment mode: nothing leaves the machine except the customer's own warehouse calls - export const ALTIMATE_ASSESSMENT_MODE = truthy("ALTIMATE_ASSESSMENT_MODE") - export const OPENCODE_DISABLE_MODELS_FETCH = truthy("OPENCODE_DISABLE_MODELS_FETCH") || ALTIMATE_ASSESSMENT_MODE + /** Read live; fails closed (any value but empty, `0` or `false` turns it on). */ + export function assessmentMode(): boolean { + const value = process.env["ALTIMATE_ASSESSMENT_MODE"]?.trim().toLowerCase() + return value !== undefined && value !== "" && value !== "0" && value !== "false" + } + export const OPENCODE_DISABLE_MODELS_FETCH = truthy("OPENCODE_DISABLE_MODELS_FETCH") || assessmentMode() // altimate_change end export const OPENCODE_DISABLE_CLAUDE_CODE = truthy("OPENCODE_DISABLE_CLAUDE_CODE") export const OPENCODE_DISABLE_CLAUDE_CODE_PROMPT = diff --git a/packages/opencode/src/provider/models.ts b/packages/opencode/src/provider/models.ts index 99aec59b3e..fd5343f1e0 100644 --- a/packages/opencode/src/provider/models.ts +++ b/packages/opencode/src/provider/models.ts @@ -164,6 +164,9 @@ export namespace ModelsDev { } export async function refresh(force = false) { + // altimate_change start — assessment mode contacts no catalog host, even when asked to refresh + if (Flag.assessmentMode()) return + // altimate_change end if (skip(force)) return ModelsDev.Data.reset() await Flock.withLock(`models-dev:${filepath}`, async () => { if (skip(force)) return ModelsDev.Data.reset() diff --git a/packages/opencode/test/altimate/assessment-zero-egress.test.ts b/packages/opencode/test/altimate/assessment-zero-egress.test.ts index a3cec8eb9f..8434f07079 100644 --- a/packages/opencode/test/altimate/assessment-zero-egress.test.ts +++ b/packages/opencode/test/altimate/assessment-zero-egress.test.ts @@ -29,7 +29,7 @@ const scratch: string[] = [] function cleanEnv(): Record { const base: Record = {} for (const [key, value] of Object.entries(process.env)) { - if (value !== undefined && !/^(NODE_ENV|BUN_TEST|VITEST|JEST_WORKER_ID|ALTIMATE_TELEMETRY_.*|ALTIMATE_ASSESSMENT_MODE|OPENCODE_.*|APPLICATIONINSIGHTS_.*)$/.test(key)) base[key] = value + if (value !== undefined && !/^(NODE_ENV|BUN_TEST|VITEST|JEST_WORKER_ID|ALTIMATE_.*|OPENCODE_.*|APPLICATIONINSIGHTS_.*|XDG_.*|CLOUDSDK_CONFIG)$/.test(key)) base[key] = value } return base } @@ -97,6 +97,8 @@ async function run(args: string[], env: Record, setup?: (home: s env: { ...base, HOME: home, + // No ripgrep on the path: without the downloader guards the CLI would fetch it from github.com. + PATH: "/usr/bin:/bin", XDG_CONFIG_HOME: path.join(home, ".config"), XDG_DATA_HOME: path.join(home, ".local/share"), XDG_CACHE_HOME: path.join(home, ".cache"), @@ -134,13 +136,6 @@ describe("assessment mode on the packaged CLI", () => { expect(result.hosts).toEqual([]) }, 120_000) - posixOnly("opt-out by project config: no host is contacted", async () => { - const result = await run(savings, { ALTIMATE_ASSESSMENT_MODE: "true" }, async (_home, project) => { - await writeFile(path.join(project, "altimate-code.json"), JSON.stringify({ telemetry: { disabled: true } })) - }) - expect(result.hosts).toEqual([]) - }, 120_000) - posixOnly("telemetry left on: no host is contacted either", async () => { const result = await run(savings, { ALTIMATE_ASSESSMENT_MODE: "true" }) expect(result.hosts).toEqual([]) @@ -148,19 +143,108 @@ describe("assessment mode on the packaged CLI", () => { posixOnly("an unreachable warehouse still contacts no other host", async () => { const result = await run(savings, assessment, async (_home, project) => { - const connections = path.join(project, "connections.json") + await mkdir(path.join(project, ".altimate-code"), { recursive: true }) await writeFile( - connections, + path.join(project, ".altimate-code", "connections.json"), JSON.stringify({ dbx: { type: "databricks", server_hostname: "127.0.0.1", http_path: "/sql/1.0/warehouses/x", access_token: "unused" } }), ) }) expect(result.hosts.filter((host) => !host.startsWith("127.0.0.1"))).toEqual([]) + // The failure comes from the connection attempt, not from a missing warehouse. + const output = JSON.parse(result.stdout).output as string + expect(output).not.toContain("none are set up") + }, 120_000) + + posixOnly("signed in to Altimate: no host is contacted", async () => { + // The ambient PATH: with a bare one the project-binding lookup is never reached, so the case would prove nothing. + const result = await run(savings, { ...assessment, PATH: process.env.PATH ?? "" }, async (home) => { + await writeFile( + path.join(home, ".altimate", "altimate.json"), + JSON.stringify({ altimateUrl: `https://${TELEMETRY_SINK}`, altimateInstanceName: "acme", altimateApiKey: "x" }), + ) + }) + expect(result.hosts).toEqual([]) + }, 120_000) + + posixOnly("a well-known remote config entry and project plugin sources: no host is contacted", async () => { + const result = await run(savings, assessment, async (home, project) => { + const data = path.join(home, ".local/share/altimate-code") + await mkdir(data, { recursive: true }) + await writeFile(path.join(data, "auth.json"), JSON.stringify({ "https://org-config.invalid": { type: "wellknown", key: "K", token: "T" } })) + await mkdir(path.join(project, ".opencode", "plugin"), { recursive: true }) + await writeFile(path.join(project, ".opencode", "plugin", "x.ts"), "export default async () => ({})\n") + }) + expect(result.hosts).toEqual([]) }, 120_000) + posixOnly("any non-empty value turns the mode on (`yes` must not fail open)", async () => { + const result = await run(savings, { ALTIMATE_ASSESSMENT_MODE: "yes", ALTIMATE_TELEMETRY_DISABLED: "true" }) + expect(result.hosts).toEqual([]) + }, 120_000) + + posixOnly("a tool daemon started without the mode does not serve a run that sets it", async () => { + const dir = await mkdtemp(path.join(tmpdir(), "assessment-daemon-")) + scratch.push(dir) + const { connectionEnvKey } = await import("../../src/altimate/tool-cli/daemon") + expect(connectionEnvKey({ ALTIMATE_ASSESSMENT_MODE: "true" })).not.toBe(connectionEnvKey({})) + }) + posixOnly("the opt-out marker and machine id are untouched", async () => { const result = await run(savings, assessment) const altimate = await readdir(path.join(result.home, ".altimate")) expect(altimate.sort()).toEqual(["machine-id"]) expect((await readFile(path.join(result.home, ".altimate", "machine-id"), "utf8")).trim()).toBe(MACHINE_ID) }, 120_000) + + posixOnly("the core ripgrep downloader refuses, and contacts no host", async () => { + const result = await probe("fixtures/assessment-rg-probe.ts", { ALTIMATE_ASSESSMENT_MODE: "true" }) + expect(result.hosts).toEqual([]) + expect(JSON.parse(result.stdout)).toMatchObject({ ok: false }) + }, 60_000) + + posixOnly("the sign-in context carries no machine id and an explicit catalog refresh contacts no host", async () => { + const on = await probe("fixtures/assessment-misc-probe.ts", { ALTIMATE_ASSESSMENT_MODE: "true" }) + expect(on.hosts).toEqual([]) + expect(JSON.parse(on.stdout)).toEqual({ hasMachineId: false }) + // Control: the same probe without the mode does mint a machine id and reach the catalog host. + const off = await probe("fixtures/assessment-misc-probe.ts", {}) + expect(JSON.parse(off.stdout)).toEqual({ hasMachineId: true }) + expect(off.hosts.some((host) => host.startsWith("models.dev"))).toBe(true) + }, 90_000) + + test.each(["true", "1", "yes", "on"])("assessment mode `%s` turns every switch on, and off values leave them off", async (value) => { + const on = JSON.parse((await probe("fixtures/assessment-flags-probe.ts", { ALTIMATE_ASSESSMENT_MODE: value })).stdout) + expect(on).toEqual({ coreMode: true, coreModelsOff: true, coreWorkspaceOff: true, mode: true, modelsOff: true }) + }, 60_000) + + test.each(["", "0", "false"])("assessment mode `%s` leaves the switches off", async (value) => { + const off = JSON.parse((await probe("fixtures/assessment-flags-probe.ts", { ALTIMATE_ASSESSMENT_MODE: value })).stdout) + expect(off).toEqual({ coreMode: false, coreModelsOff: false, coreWorkspaceOff: false, mode: false, modelsOff: false }) + }, 60_000) }) + +/** Runs a fixture script in a fresh home with the mode's environment, behind the logging proxy. */ +async function probe(fixture: string, env: Record) { + const home = await mkdtemp(path.join(tmpdir(), "assessment-probe-")) + scratch.push(home) + seen = [] + const proc = Bun.spawn([process.execPath, "run", path.join(import.meta.dir, fixture)], { + stdout: "pipe", + stderr: "pipe", + stdin: "ignore", + timeout: 60_000, + env: { + PATH: "/usr/bin:/bin", + HOME: home, + XDG_DATA_HOME: path.join(home, "d"), + XDG_CACHE_HOME: path.join(home, "c"), + XDG_STATE_HOME: path.join(home, "s"), + XDG_CONFIG_HOME: path.join(home, "cf"), + HTTPS_PROXY: `http://127.0.0.1:${proxyPort}`, + HTTP_PROXY: `http://127.0.0.1:${proxyPort}`, + ...env, + }, + }) + const [stdout] = await Promise.all([new Response(proc.stdout).text(), proc.exited]) + return { stdout: stdout.trim().split("\n").pop() ?? "", hosts: [...new Set(seen)].sort() } +} diff --git a/packages/opencode/test/altimate/fixtures/assessment-flags-probe.ts b/packages/opencode/test/altimate/fixtures/assessment-flags-probe.ts new file mode 100644 index 0000000000..886e37de2c --- /dev/null +++ b/packages/opencode/test/altimate/fixtures/assessment-flags-probe.ts @@ -0,0 +1,12 @@ +import { Flag as CoreFlag } from "@opencode-ai/core/flag/flag" +import { Flag } from "../../../src/flag/flag" + +console.log( + JSON.stringify({ + coreMode: CoreFlag.ALTIMATE_ASSESSMENT_MODE, + coreModelsOff: CoreFlag.OPENCODE_DISABLE_MODELS_FETCH, + coreWorkspaceOff: CoreFlag.ALTIMATE_DISABLE_WORKSPACE, + mode: Flag.assessmentMode(), + modelsOff: Flag.OPENCODE_DISABLE_MODELS_FETCH, + }), +) diff --git a/packages/opencode/test/altimate/fixtures/assessment-misc-probe.ts b/packages/opencode/test/altimate/fixtures/assessment-misc-probe.ts new file mode 100644 index 0000000000..5cd82f47a7 --- /dev/null +++ b/packages/opencode/test/altimate/fixtures/assessment-misc-probe.ts @@ -0,0 +1,11 @@ +import fs from "node:fs" +import os from "node:os" +import path from "node:path" +import { buildCliContext } from "../../../src/altimate/plugin/altimate" +import { ModelsDev } from "../../../src/provider/models" + +const dir = fs.mkdtempSync(path.join(os.tmpdir(), "assessment-mid-")) +const encoded = await buildCliContext(path.join(dir, "machine-id")) +const context = JSON.parse(Buffer.from(encoded, "base64url").toString("utf8")) +await ModelsDev.refresh(true) +console.log(JSON.stringify({ hasMachineId: "machine_id" in context })) diff --git a/packages/opencode/test/altimate/fixtures/assessment-rg-probe.ts b/packages/opencode/test/altimate/fixtures/assessment-rg-probe.ts new file mode 100644 index 0000000000..bcbf09402f --- /dev/null +++ b/packages/opencode/test/altimate/fixtures/assessment-rg-probe.ts @@ -0,0 +1,6 @@ +import { RipgrepBinary } from "@opencode-ai/core/ripgrep/binary" +import { Effect } from "effect" +const out = await Effect.runPromise( + Effect.gen(function* () { const s = yield* RipgrepBinary.Service; return yield* s.filepath }).pipe(Effect.provide(RipgrepBinary.defaultLayer), Effect.result), +) +console.log(JSON.stringify({ ok: out._tag === "Success", text: String((out as any).failure ?? (out as any).success).slice(0, 200) })) From 3301c3208394cd488053f8b3b9f9c670c6639c6c Mon Sep 17 00:00:00 2001 From: Shreyas Telkar Date: Tue, 17 Nov 2026 20:47:13 -0800 Subject: [PATCH 11/34] fix: assessment mode keeps the well-known token for substitution; flag tests skip on Windows The well-known auth entry's token is still registered for `{env:...}` substitution; only the remote-config fetch is skipped. The two flag-value test groups use the same POSIX-only guard as the rest of the file, and the opencode flag reuses core's fail-closed parser. Co-Authored-By: Claude Sonnet 5.5 --- packages/opencode/src/config/config.ts | 6 +++--- packages/opencode/src/flag/flag.ts | 5 ++--- .../opencode/test/altimate/assessment-zero-egress.test.ts | 4 ++-- 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/packages/opencode/src/config/config.ts b/packages/opencode/src/config/config.ts index b5db4577d7..fd0e28af80 100644 --- a/packages/opencode/src/config/config.ts +++ b/packages/opencode/src/config/config.ts @@ -535,12 +535,12 @@ export const layer = Layer.effect( } for (const [key, value] of Object.entries(auth)) { - // altimate_change start — assessment mode fetches no remote config - if (value.type === "wellknown" && Flag.ALTIMATE_ASSESSMENT_MODE) continue - // altimate_change end if (value.type === "wellknown") { const url = key.replace(/\/+$/, "") authEnv[value.key] = value.token + // altimate_change start — assessment mode fetches no remote config; the token stays available for substitution + if (Flag.ALTIMATE_ASSESSMENT_MODE) continue + // altimate_change end const wellknownURL = `${url}/.well-known/opencode` yield* Effect.logDebug("fetching remote config", { url: wellknownURL }) const wellknown = yield* fetchRemoteJson(wellknownURL, undefined, ConfigV1.WellKnown, url) diff --git a/packages/opencode/src/flag/flag.ts b/packages/opencode/src/flag/flag.ts index cb97cc06a3..f01cb4d845 100644 --- a/packages/opencode/src/flag/flag.ts +++ b/packages/opencode/src/flag/flag.ts @@ -5,7 +5,7 @@ // read as-is. // The rule itself lives in core's `flag/flag.ts` (`env`), which the core `Flag` object, this // namespace and the Effect `Config`-backed services (`effect/config-service.ts`) all share. -import { env as read } from "@opencode-ai/core/flag/flag" +import { env as read, killSwitchSet } from "@opencode-ai/core/flag/flag" // altimate_change end function truthy(key: string) { @@ -116,8 +116,7 @@ export namespace Flag { // altimate_change start — assessment mode: nothing leaves the machine except the customer's own warehouse calls /** Read live; fails closed (any value but empty, `0` or `false` turns it on). */ export function assessmentMode(): boolean { - const value = process.env["ALTIMATE_ASSESSMENT_MODE"]?.trim().toLowerCase() - return value !== undefined && value !== "" && value !== "0" && value !== "false" + return killSwitchSet(process.env["ALTIMATE_ASSESSMENT_MODE"]) } export const OPENCODE_DISABLE_MODELS_FETCH = truthy("OPENCODE_DISABLE_MODELS_FETCH") || assessmentMode() // altimate_change end diff --git a/packages/opencode/test/altimate/assessment-zero-egress.test.ts b/packages/opencode/test/altimate/assessment-zero-egress.test.ts index 8434f07079..fe906b2960 100644 --- a/packages/opencode/test/altimate/assessment-zero-egress.test.ts +++ b/packages/opencode/test/altimate/assessment-zero-egress.test.ts @@ -212,12 +212,12 @@ describe("assessment mode on the packaged CLI", () => { expect(off.hosts.some((host) => host.startsWith("models.dev"))).toBe(true) }, 90_000) - test.each(["true", "1", "yes", "on"])("assessment mode `%s` turns every switch on, and off values leave them off", async (value) => { + posixOnly.each(["true", "1", "yes", "on"])("assessment mode `%s` turns every switch on, and off values leave them off", async (value) => { const on = JSON.parse((await probe("fixtures/assessment-flags-probe.ts", { ALTIMATE_ASSESSMENT_MODE: value })).stdout) expect(on).toEqual({ coreMode: true, coreModelsOff: true, coreWorkspaceOff: true, mode: true, modelsOff: true }) }, 60_000) - test.each(["", "0", "false"])("assessment mode `%s` leaves the switches off", async (value) => { + posixOnly.each(["", "0", "false"])("assessment mode `%s` leaves the switches off", async (value) => { const off = JSON.parse((await probe("fixtures/assessment-flags-probe.ts", { ALTIMATE_ASSESSMENT_MODE: value })).stdout) expect(off).toEqual({ coreMode: false, coreModelsOff: false, coreWorkspaceOff: false, mode: false, modelsOff: false }) }, 60_000) From 6e0c010cb66c8ee6baa2b40d8105d7bfc9d64fbe Mon Sep 17 00:00:00 2001 From: Shreyas Telkar Date: Tue, 17 Nov 2026 20:51:34 -0800 Subject: [PATCH 12/34] fix: assessment mode keeps local sign-out cleanup and makes no active-account calls `signOut()` checks for saved credentials without the assessment-mode gate, so signing out still removes them. The active-account config fetch and token refresh are skipped in the mode. The packaged-binary tests run with an empty PATH directory so an installed ripgrep cannot hide a missing download guard. Co-Authored-By: Claude Sonnet 5.5 --- packages/opencode/src/altimate/api/client.ts | 11 ++++++++--- packages/opencode/src/config/config.ts | 3 ++- .../test/altimate/assessment-zero-egress.test.ts | 6 +++--- 3 files changed, 13 insertions(+), 7 deletions(-) diff --git a/packages/opencode/src/altimate/api/client.ts b/packages/opencode/src/altimate/api/client.ts index 9a6e5278ea..943cf54a90 100644 --- a/packages/opencode/src/altimate/api/client.ts +++ b/packages/opencode/src/altimate/api/client.ts @@ -82,11 +82,16 @@ export namespace AltimateApi { return { "x-altimate-client": `altimate-code/${InstallationVersion}` } } + /** Whether credentials exist, in assessment mode too: signing out must still find and remove them. */ + async function credentialsPresent(): Promise { + if (currentCredentials()) return true + return Filesystem.exists(credentialsPath()) + } + export async function isConfigured(): Promise { // Assessment mode talks to no Altimate service: to every caller the user is signed out. if (Flag.assessmentMode()) return false - if (currentCredentials()) return true - return Filesystem.exists(credentialsPath()) + return credentialsPresent() } /** A `${env:NAME}` in the credentials file names a variable that isn't set. */ @@ -192,7 +197,7 @@ export namespace AltimateApi { * `projectDir` loads. Fails without deleting the file when the key can't be read: the file holds the * only copy, and without it leftover MCP entries can never be matched. */ export async function signOut(projectDir: string): Promise> { - if (!(await isConfigured())) { + if (!(await credentialsPresent())) { await withdrawClaudeSkills(projectDir) await forgetTelemetryIdentity() return [] diff --git a/packages/opencode/src/config/config.ts b/packages/opencode/src/config/config.ts index fd0e28af80..03565568a9 100644 --- a/packages/opencode/src/config/config.ts +++ b/packages/opencode/src/config/config.ts @@ -660,7 +660,8 @@ export const layer = Layer.effect( const activeAccount = Option.getOrUndefined( yield* accountSvc.active().pipe(Effect.catch(() => Effect.succeed(Option.none()))), ) - if (activeAccount?.active_org_id) { + // altimate_change — assessment mode makes no account calls (config fetch, token refresh) + if (activeAccount?.active_org_id && !Flag.ALTIMATE_ASSESSMENT_MODE) { const accountID = activeAccount.id const orgID = activeAccount.active_org_id const url = activeAccount.url diff --git a/packages/opencode/test/altimate/assessment-zero-egress.test.ts b/packages/opencode/test/altimate/assessment-zero-egress.test.ts index fe906b2960..0e2e430b8b 100644 --- a/packages/opencode/test/altimate/assessment-zero-egress.test.ts +++ b/packages/opencode/test/altimate/assessment-zero-egress.test.ts @@ -97,8 +97,8 @@ async function run(args: string[], env: Record, setup?: (home: s env: { ...base, HOME: home, - // No ripgrep on the path: without the downloader guards the CLI would fetch it from github.com. - PATH: "/usr/bin:/bin", + // A PATH with nothing on it, so no ripgrep is found: without the downloader guards the CLI would fetch it from github.com. + PATH: path.join(home, "empty-bin"), XDG_CONFIG_HOME: path.join(home, ".config"), XDG_DATA_HOME: path.join(home, ".local/share"), XDG_CACHE_HOME: path.join(home, ".cache"), @@ -234,7 +234,7 @@ async function probe(fixture: string, env: Record) { stdin: "ignore", timeout: 60_000, env: { - PATH: "/usr/bin:/bin", + PATH: path.join(home, "empty-bin"), HOME: home, XDG_DATA_HOME: path.join(home, "d"), XDG_CACHE_HOME: path.join(home, "c"), From 4e3799c7f73dfba087be3b24d701ddf292d28b34 Mon Sep 17 00:00:00 2001 From: Shreyas Telkar Date: Tue, 17 Nov 2026 21:01:39 -0800 Subject: [PATCH 13/34] fix: assessment mode installs no npm plugin in the plugin loader The config preinstall was already skipped; the loader's on-demand install and the plugin target resolver are now skipped too. Local file plugins still load. Co-Authored-By: Claude Sonnet 5.5 --- packages/opencode/src/plugin/index.ts | 6 ++++++ packages/opencode/src/plugin/shared.ts | 4 ++++ 2 files changed, 10 insertions(+) diff --git a/packages/opencode/src/plugin/index.ts b/packages/opencode/src/plugin/index.ts index 7bd2e7d5c8..ab68bd14cb 100644 --- a/packages/opencode/src/plugin/index.ts +++ b/packages/opencode/src/plugin/index.ts @@ -134,6 +134,12 @@ export namespace Plugin { // ignore old codex plugin since it is supported first party now if (plugin.includes("opencode-openai-codex-auth") || plugin.includes("opencode-copilot-auth")) continue log.info("loading plugin", { path: plugin }) + // altimate_change start — assessment mode installs no npm plugin; local file plugins still load + if (!plugin.startsWith("file://") && Flag.assessmentMode()) { + log.warn("skipping npm plugin: assessment mode installs nothing", { path: plugin }) + continue + } + // altimate_change end if (!plugin.startsWith("file://")) { const lastAtIndex = plugin.lastIndexOf("@") const pkg = lastAtIndex > 0 ? plugin.substring(0, lastAtIndex) : plugin diff --git a/packages/opencode/src/plugin/shared.ts b/packages/opencode/src/plugin/shared.ts index ec7b99c40b..0484936076 100644 --- a/packages/opencode/src/plugin/shared.ts +++ b/packages/opencode/src/plugin/shared.ts @@ -1,4 +1,5 @@ import path from "path" +import { Flag } from "../flag/flag" import { fileURLToPath, pathToFileURL } from "url" import npa from "npm-package-arg" import semver from "semver" @@ -208,6 +209,9 @@ export async function checkPluginCompatibility(target: string, opencodeVersion: export async function resolvePluginTarget(spec: string) { if (isPathPluginSpec(spec)) return resolvePathPluginTarget(spec) + // altimate_change start — assessment mode installs no npm plugin + if (Flag.assessmentMode()) throw new Error("npm plugins are not installed in assessment mode") + // altimate_change end const hit = parse(spec) const pkg = hit?.name && hit.raw === hit.name ? `${hit.name}@latest` : spec const result = await Npm.add(pkg) From cd5ed304ef7b5440560c0015624f24871b3218f1 Mon Sep 17 00:00:00 2001 From: Shreyas Telkar Date: Wed, 18 Nov 2026 02:27:27 -0800 Subject: [PATCH 14/34] docs: serverless SQL warehouses get the over-sized check The tool sends every SQL warehouse to altimate-core, and the over-sized rule does not exclude serverless ones, so "serverless warehouses are not covered" was wrong. Only Databricks Apps spend is outside the check. Co-Authored-By: Claude Opus 5.5 --- docs/docs/data-engineering/tools/finops-tools.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/docs/data-engineering/tools/finops-tools.md b/docs/docs/data-engineering/tools/finops-tools.md index 4b98ca7415..13c78a32b6 100644 --- a/docs/docs/data-engineering/tools/finops-tools.md +++ b/docs/docs/data-engineering/tools/finops-tools.md @@ -189,7 +189,7 @@ Check a Databricks workspace for savings with Altimate's rules, on a Databricks The tool measures each resource's usage; altimate-core decides what qualifies and what it saves. Prices are list prices. Each resource is counted once in the total, and each flagged warehouse also gets what it would cost at every smaller size and as serverless. If the connection cannot read the cluster or job tables, that check is named as not checked and the warehouse checks still report; the warehouse tables themselves are required. -**What it does not assess.** Serverless warehouses and Databricks Apps spend are not covered. A SQL warehouse's cost is its billed usage for the six full days before the window's last day, counting only records Databricks had ingested before that day, spread over the seven-day window as Altimate's pipeline does, so it reads lower than seven days of billing (on one measured workspace, about 7% lower). A cluster's or job's cost is its billed usage for the seven full days before the window's last day. The connection needs the `@databricks/sql` driver installed. +**What it does not assess.** Databricks Apps spend is not covered, and a serverless SQL warehouse gets only the Over-Sized Warehouse check. A SQL warehouse's cost is its billed usage for the six full days before the window's last day, counting only records Databricks had ingested before that day, spread over the seven-day window as Altimate's pipeline does, so it reads lower than seven days of billing (on one measured workspace, about 7% lower). A cluster's or job's cost is its billed usage for the seven full days before the window's last day. The connection needs the `@databricks/sql` driver installed. **Assessment mode.** Set `ALTIMATE_ASSESSMENT_MODE=true` to keep the run from contacting anything but your warehouse. It sends no telemetry (not even the one-time opt-out event), treats you as signed out of Altimate (no workspace or project-binding lookups), fetches no remote config and installs no plugin dependencies, and downloads neither the model catalog (models.dev) nor ripgrep (github.com). Any value but empty, `0` or `false` turns it on, and a tool daemon started without it is not reused. It is measured for `tool run`; other commands (the update check, language-server downloads) are not covered. Core's compiled analysis is not audited for network use. Without it, `ALTIMATE_TELEMETRY_DISABLED=true` alone still lets the CLI send that opt-out event and fetch those. The Claude Code `/savings` command sets it for you. Tool results shown in a Claude Code conversation reach its model provider; assessment mode does not change that. From db658f4535a643908f19c3a9fd3e307e9cff52a2 Mon Sep 17 00:00:00 2001 From: Shreyas Telkar Date: Wed, 18 Nov 2026 02:36:06 -0800 Subject: [PATCH 15/34] fix: assessment mode switches core's own telemetry off before the savings rules run altimate-core sends its usage event unless its opt-out variables or `initSdk` say otherwise; it does not read assessment mode, and a variable set from the CLI does not reach it. With only `ALTIMATE_ASSESSMENT_MODE=true`, the rules call started core's telemetry worker. The rules loader now calls `initSdk` with telemetry and failure reports off in assessment mode (it only stores the setting). Probed with network denied and a throwaway home: the rules called directly wrote core's machine id (its telemetry worker started); through the loader they did not. Co-Authored-By: Claude Opus 5.5 --- .../native/finops/databricks-savings.ts | 17 ++++++++++---- .../finops-databricks-savings.test.ts | 22 +++++++++++++++++++ 2 files changed, 35 insertions(+), 4 deletions(-) diff --git a/packages/opencode/src/altimate/native/finops/databricks-savings.ts b/packages/opencode/src/altimate/native/finops/databricks-savings.ts index 44dd64de3b..93e711ce38 100644 --- a/packages/opencode/src/altimate/native/finops/databricks-savings.ts +++ b/packages/opencode/src/altimate/native/finops/databricks-savings.ts @@ -7,6 +7,7 @@ */ import * as core from "@altimateai/altimate-core" +import { Flag } from "@opencode-ai/core/flag/flag" import * as Registry from "../connections/registry" import { resolveFinopsWarehouse } from "./warehouse-resolver" import type { DatabricksSavingsParams, DatabricksSavingsResult, DatabricksSavingsReport } from "../types" @@ -282,10 +283,18 @@ ORDER BY jb.job_all_purpose_cost DESC /** Core's rules: a `DatabricksSavingsInput` as JSON in, the report out. */ export type RulesEngine = (input: string) => DatabricksSavingsReport -/** The installed core's rules, or null when its binary predates them. */ -function rulesEngine(): RulesEngine | null { - const found = (core as unknown as { databricksSavings?: unknown }).databricksSavings - return typeof found === "function" ? (found as RulesEngine) : null +type CoreModule = { databricksSavings?: unknown; initSdk?: (...args: [string, string, string, boolean, boolean]) => void } + +/** + * The installed core's rules, or null when its binary predates them. In assessment mode core's own + * usage event is switched off first: core reads its telemetry opt-out variables or `initSdk`, not + * assessment mode, and a variable set from here does not reach it. `initSdk` only stores the setting. + */ +export function rulesEngine(lib: CoreModule = core as unknown as CoreModule): RulesEngine | null { + const found = lib.databricksSavings + if (typeof found !== "function") return null + if (Flag.ALTIMATE_ASSESSMENT_MODE) lib.initSdk?.("", "", "", false, false) + return found as RulesEngine } /** The seven-day window the facts cover: it ends at 06:00 UTC on `end`, the latest one that has passed. */ diff --git a/packages/opencode/test/altimate/finops-databricks-savings.test.ts b/packages/opencode/test/altimate/finops-databricks-savings.test.ts index 577ab5fcf5..4d89e483f1 100644 --- a/packages/opencode/test/altimate/finops-databricks-savings.test.ts +++ b/packages/opencode/test/altimate/finops-databricks-savings.test.ts @@ -7,6 +7,7 @@ import { DATABRICKS_JOB_FACTS_SQL, DATABRICKS_WAREHOUSE_FACTS_SQL, databricksSavings, + rulesEngine, savingsWindow, toClusterFacts, toFacts, @@ -147,6 +148,27 @@ describe("finops_databricks_savings tool", () => { describe("databricks savings handler", () => { afterEach(() => Registry.reset()) + test("in assessment mode the rules come with core's own telemetry switched off; otherwise core is left as it is", () => { + const saved = process.env["ALTIMATE_ASSESSMENT_MODE"] + const calls: unknown[][] = [] + const lib = { databricksSavings: () => ({}), initSdk: (...args: unknown[]) => void calls.push(args) } + try { + process.env["ALTIMATE_ASSESSMENT_MODE"] = "true" + expect(rulesEngine(lib)).toBe(lib.databricksSavings as never) + // Telemetry off, failure reports off; `initSdk` only stores the setting. + expect(calls).toEqual([["", "", "", false, false]]) + // A core without the rules is not touched: the tool refuses before any rules call. + expect(rulesEngine({ initSdk: lib.initSdk })).toBeNull() + expect(calls.length).toBe(1) + delete process.env["ALTIMATE_ASSESSMENT_MODE"] + expect(rulesEngine(lib)).toBe(lib.databricksSavings as never) + expect(calls.length).toBe(1) + } finally { + if (saved === undefined) delete process.env["ALTIMATE_ASSESSMENT_MODE"] + else process.env["ALTIMATE_ASSESSMENT_MODE"] = saved + } + }) + test("the window is the seven days to the latest 06:00 UTC that has passed", () => { expect(savingsWindow(Date.parse("2026-11-12T10:00:00Z"))).toEqual({ start: "2026-11-05", end: "2026-11-12" }) expect(savingsWindow(Date.parse("2026-11-12T06:00:00Z"))).toEqual({ start: "2026-11-05", end: "2026-11-12" }) From 41fce2a435ba1855c0acadccc6488ec24b51010b Mon Sep 17 00:00:00 2001 From: Shreyas Telkar Date: Wed, 18 Nov 2026 02:45:28 -0800 Subject: [PATCH 16/34] fix: assessment mode silences altimate-core's own telemetry for every command The savings rules loader switched core's usage event off, but other core-backed tools (`sql_analyze` through `core.lint`, `checkSemantics`, `scanSql`) still started core's telemetry worker in assessment mode. The CLI middleware now awaits `CoreTelemetry.silence()` (core's `initSdk` with telemetry and failure reports off; it only stores the setting) before any command runs. Checked on the source CLI with network denied and a fresh home: `sql_analyze` in assessment mode wrote core's machine id without the middleware call and did not with it. The packaged-CLI suite gains that case (20 pass). Co-Authored-By: Claude Opus 5.5 --- .../src/altimate/native/core-telemetry.ts | 18 ++++++++++++++++++ packages/opencode/src/index.ts | 3 +++ .../altimate/assessment-zero-egress.test.ts | 15 +++++++++++++++ 3 files changed, 36 insertions(+) create mode 100644 packages/opencode/src/altimate/native/core-telemetry.ts diff --git a/packages/opencode/src/altimate/native/core-telemetry.ts b/packages/opencode/src/altimate/native/core-telemetry.ts new file mode 100644 index 0000000000..f49beca453 --- /dev/null +++ b/packages/opencode/src/altimate/native/core-telemetry.ts @@ -0,0 +1,18 @@ +/** + * altimate-core sends its own usage event (operation, duration, machine id) unless its opt-out + * variables or `initSdk` say otherwise. It does not read assessment mode, and a variable set from + * this process does not reach it, so assessment mode switches it off through `initSdk`, which only + * stores the setting. A core that cannot load has nothing to send. + */ +export async function silence(): Promise { + try { + const core = (await import("@altimateai/altimate-core")) as unknown as { + initSdk?: (apiKey: string, tenant: string, backendUrl: string, reportFailures: boolean, telemetry: boolean) => void + } + core.initSdk?.("", "", "", false, false) + } catch { + // No core binary on this platform: no core call can run, so none can send. + } +} + +export * as CoreTelemetry from "./core-telemetry" diff --git a/packages/opencode/src/index.ts b/packages/opencode/src/index.ts index 3f52c1a655..626208edb5 100644 --- a/packages/opencode/src/index.ts +++ b/packages/opencode/src/index.ts @@ -71,6 +71,7 @@ import { PluginCommand } from "./cli/cmd/plug" import { Heap } from "./cli/heap" // altimate_change start - telemetry import import { Telemetry } from "./telemetry" +import { CoreTelemetry } from "./altimate/native/core-telemetry" // altimate_change end // altimate_change start - welcome banner import { showWelcomeBannerIfNeeded } from "./cli/welcome" @@ -217,6 +218,8 @@ let cli = yargs(args) : undefined, ) if (!noNetwork) Telemetry.init().catch(() => {}) + // Assessment mode also silences altimate-core's own usage event; awaited so no command reaches core first. + if (Flag.ALTIMATE_ASSESSMENT_MODE) await CoreTelemetry.silence() // altimate_change end }) .usage("") diff --git a/packages/opencode/test/altimate/assessment-zero-egress.test.ts b/packages/opencode/test/altimate/assessment-zero-egress.test.ts index 0e2e430b8b..f12d18c950 100644 --- a/packages/opencode/test/altimate/assessment-zero-egress.test.ts +++ b/packages/opencode/test/altimate/assessment-zero-egress.test.ts @@ -189,6 +189,21 @@ describe("assessment mode on the packaged CLI", () => { expect(connectionEnvKey({ ALTIMATE_ASSESSMENT_MODE: "true" })).not.toBe(connectionEnvKey({})) }) + posixOnly("altimate-core's own usage event stays off too: a core-backed tool starts no core telemetry", async () => { + // Core writes ~/.altimate/machine-id when its telemetry worker starts, and its client does not + // go through the proxy above, so the machine id is what shows it. Telemetry is left on here: + // assessment mode alone has to switch core off. + const result = await run( + ["tool", "run", "sql_analyze", "--args", JSON.stringify({ sql: "select a from t where b = 1", dialect: "snowflake" }), "--json", "--no-daemon"], + { ALTIMATE_ASSESSMENT_MODE: "true" }, + (home) => rm(path.join(home, ".altimate", "machine-id")), + ) + expect(JSON.parse(result.stdout).status).toBe("ok") + await new Promise((resolve) => setTimeout(resolve, 2_000)) + expect(await readdir(path.join(result.home, ".altimate"))).toEqual([]) + expect(result.hosts).toEqual([]) + }, 120_000) + posixOnly("the opt-out marker and machine id are untouched", async () => { const result = await run(savings, assessment) const altimate = await readdir(path.join(result.home, ".altimate")) From 2ab21ecd8638a3e3e7db2a9307d0cfd2441bee1d Mon Sep 17 00:00:00 2001 From: Shreyas Telkar Date: Wed, 18 Nov 2026 03:02:54 -0800 Subject: [PATCH 17/34] fix: assessment mode silences core before `mcp serve` returns from the middleware `mcp serve` returns early from the CLI middleware, before the core-telemetry switch, so the long-lived MCP server that serves core-backed tools kept core's usage event on in assessment mode. The switch now runs before that return. The packaged-CLI case drops a sleep that only ran after the process had exited. Co-Authored-By: Claude Opus 5.5 --- packages/opencode/src/index.ts | 6 ++++-- .../opencode/test/altimate/assessment-zero-egress.test.ts | 1 - 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/packages/opencode/src/index.ts b/packages/opencode/src/index.ts index 626208edb5..f3f59bd787 100644 --- a/packages/opencode/src/index.ts +++ b/packages/opencode/src/index.ts @@ -162,6 +162,10 @@ let cli = yargs(args) if (opts.pure) { process.env.OPENCODE_PURE = "1" } + // altimate_change start — assessment mode silences altimate-core's own usage event before any + // command can reach core, `mcp serve` (which returns just below) included + if (Flag.ALTIMATE_ASSESSMENT_MODE) await CoreTelemetry.silence() + // altimate_change end // altimate_change start — MCP stdio needs no banner, telemetry startup or agent initialization if (opts._[0] === "mcp" && opts._[1] === "serve") return // altimate_change end @@ -218,8 +222,6 @@ let cli = yargs(args) : undefined, ) if (!noNetwork) Telemetry.init().catch(() => {}) - // Assessment mode also silences altimate-core's own usage event; awaited so no command reaches core first. - if (Flag.ALTIMATE_ASSESSMENT_MODE) await CoreTelemetry.silence() // altimate_change end }) .usage("") diff --git a/packages/opencode/test/altimate/assessment-zero-egress.test.ts b/packages/opencode/test/altimate/assessment-zero-egress.test.ts index f12d18c950..51b99a455b 100644 --- a/packages/opencode/test/altimate/assessment-zero-egress.test.ts +++ b/packages/opencode/test/altimate/assessment-zero-egress.test.ts @@ -199,7 +199,6 @@ describe("assessment mode on the packaged CLI", () => { (home) => rm(path.join(home, ".altimate", "machine-id")), ) expect(JSON.parse(result.stdout).status).toBe("ok") - await new Promise((resolve) => setTimeout(resolve, 2_000)) expect(await readdir(path.join(result.home, ".altimate"))).toEqual([]) expect(result.hosts).toEqual([]) }, 120_000) From 73683e17d4b1cc2006b13ba68e35415da393d716 Mon Sep 17 00:00:00 2001 From: Shreyas Telkar Date: Wed, 18 Nov 2026 03:09:05 -0800 Subject: [PATCH 18/34] test: the middleware test passes the assessment flag and core switch, and checks mcp serve is covered The test rebuilds the CLI middleware with only `Heap` in scope; the core-telemetry switch now reads `Flag` before the `mcp serve` return, so it threw. It passes `Flag` and a `CoreTelemetry` stub, and a second run with assessment mode on asserts the switch ran once before `mcp serve` returned (it fails if the switch moves below the return). Co-Authored-By: Claude Opus 5.5 --- .../test/altimate/mcp-server/stdio.test.ts | 25 +++++++++++++------ 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/packages/opencode/test/altimate/mcp-server/stdio.test.ts b/packages/opencode/test/altimate/mcp-server/stdio.test.ts index 9e1cf510b3..7b671894c0 100644 --- a/packages/opencode/test/altimate/mcp-server/stdio.test.ts +++ b/packages/opencode/test/altimate/mcp-server/stdio.test.ts @@ -202,13 +202,24 @@ test("mcp serve skips CLI middleware before starting the agent heap service", as const body = source.slice(start + opening.length, end) const code = new Bun.Transpiler({ loader: "ts" }).transformSync(`async function middleware(opts: any) {${body}\n}`) let starts = 0 - const middleware = new Function("Heap", `${code}; return middleware`)({ - start() { - starts++ - throw new Error("Agent heap initialization reached by MCP server") - }, - }) - await middleware({ _: ["mcp", "serve"] }) + let silenced = 0 + const build = (assessment: boolean) => + new Function("Heap", "Flag", "CoreTelemetry", `${code}; return middleware`)( + { + start() { + starts++ + throw new Error("Agent heap initialization reached by MCP server") + }, + }, + { ALTIMATE_ASSESSMENT_MODE: assessment }, + { silence: async () => void silenced++ }, + ) + await build(false)({ _: ["mcp", "serve"] }) + expect(starts).toBe(0) + expect(silenced).toBe(0) + // Assessment mode silences altimate-core's own telemetry before `mcp serve` returns. + await build(true)({ _: ["mcp", "serve"] }) expect(starts).toBe(0) + expect(silenced).toBe(1) }) // altimate_change end From 6d1ee99d03597524310d746be0d1b2661c43e4de Mon Sep 17 00:00:00 2001 From: Shreyas Telkar Date: Wed, 18 Nov 2026 13:15:18 -0800 Subject: [PATCH 19/34] feat: the savings check runs only with `confirm: true`, and stops after 9 minutes - `finops_databricks_savings` runs nothing without `confirm: true` and tells the agent to ask the user first: a run can start a stopped SQL warehouse, which bills. That answer is not recorded as a failed run. - The check stops after 9 minutes, cancelling any query still running on the warehouse; a query is not started once the time is used up. Stopping the session cancels the running query and starts no other. Timeouts fail the check, or skip an optional check, with a plain reason. - The Databricks driver gains an opt-in `cancelAfterMs`: past it, or when `signal` aborts, it cancels the statement on the warehouse, including one the server accepts only after the deadline, and waits at most 30 s for the cancel. Calls without it, including every `executeQuery` call with its default `queryTimeoutMs`, run as before. - `tool run` aborts its tool on SIGINT/SIGTERM (a second signal exits at once), and the tool daemon aborts a run whose client disconnects, so a stopped run cancels its query. Co-Authored-By: Claude Opus 5.5 --- .../data-engineering/tools/finops-tools.md | 4 +- packages/drivers/src/databricks.ts | 85 ++++++++++- packages/drivers/src/types.ts | 4 + .../drivers/test/databricks-deadline.test.ts | 144 ++++++++++++++++++ .../native/finops/databricks-savings.ts | 28 +++- .../src/altimate/native/finops/register.ts | 4 +- .../opencode/src/altimate/tool-cli/daemon.ts | 17 ++- .../src/altimate/tool-cli/stop-signals.ts | 29 ++++ .../tools/finops-databricks-savings.ts | 29 +++- packages/opencode/src/cli/cmd/tool.ts | 5 +- .../altimate/assessment-zero-egress.test.ts | 9 +- .../finops-databricks-savings.test.ts | 116 +++++++++++++- .../test/altimate/tool-cli/daemon.test.ts | 28 ++++ .../altimate/tool-cli/stop-signals.test.ts | 32 ++++ 14 files changed, 509 insertions(+), 25 deletions(-) create mode 100644 packages/drivers/test/databricks-deadline.test.ts create mode 100644 packages/opencode/src/altimate/tool-cli/stop-signals.ts create mode 100644 packages/opencode/test/altimate/tool-cli/stop-signals.test.ts diff --git a/docs/docs/data-engineering/tools/finops-tools.md b/docs/docs/data-engineering/tools/finops-tools.md index 13c78a32b6..4820ad1bae 100644 --- a/docs/docs/data-engineering/tools/finops-tools.md +++ b/docs/docs/data-engineering/tools/finops-tools.md @@ -191,10 +191,12 @@ The tool measures each resource's usage; altimate-core decides what qualifies an **What it does not assess.** Databricks Apps spend is not covered, and a serverless SQL warehouse gets only the Over-Sized Warehouse check. A SQL warehouse's cost is its billed usage for the six full days before the window's last day, counting only records Databricks had ingested before that day, spread over the seven-day window as Altimate's pipeline does, so it reads lower than seven days of billing (on one measured workspace, about 7% lower). A cluster's or job's cost is its billed usage for the seven full days before the window's last day. The connection needs the `@databricks/sql` driver installed. +**Asks first, and stops long queries.** The check runs only when called with `confirm: true`; without it, the agent is told to ask you first, since a run can start a stopped warehouse that then bills. It runs three queries and stops after 9 minutes, cancelling any query still running on the warehouse; stopping the session, or the `tool run` process, cancels the running query too. The Claude Code `/savings` command passes `confirm: true`, since typing it is the go-ahead. + **Assessment mode.** Set `ALTIMATE_ASSESSMENT_MODE=true` to keep the run from contacting anything but your warehouse. It sends no telemetry (not even the one-time opt-out event), treats you as signed out of Altimate (no workspace or project-binding lookups), fetches no remote config and installs no plugin dependencies, and downloads neither the model catalog (models.dev) nor ripgrep (github.com). Any value but empty, `0` or `false` turns it on, and a tool daemon started without it is not reused. It is measured for `tool run`; other commands (the update check, language-server downloads) are not covered. Core's compiled analysis is not audited for network use. Without it, `ALTIMATE_TELEMETRY_DISABLED=true` alone still lets the CLI send that opt-out event and fetch those. The Claude Code `/savings` command sets it for you. Tool results shown in a Claude Code conversation reach its model provider; assessment mode does not change that. ``` -> finops_databricks_savings --warehouse prod_dbx +> finops_databricks_savings --warehouse prod_dbx --confirm Databricks savings: 7 SQL warehouses, 3 all-purpose clusters, 12 jobs checked, the 7 days to 2026-11-17, at list price. Total: $85.17–$106 a month on 1 resource, each counted once at its largest saving. diff --git a/packages/drivers/src/databricks.ts b/packages/drivers/src/databricks.ts index 3c9eee1f19..15652478f8 100644 --- a/packages/drivers/src/databricks.ts +++ b/packages/drivers/src/databricks.ts @@ -3,6 +3,8 @@ */ import type { ConnectionConfig, Connector, ConnectorResult, ExecuteOptions, SchemaColumn } from "./types" +import { QueryTimeoutError } from "./types" +import { MAX_TIMER_MS } from "./query-timeout" import { loadOptionalDriver } from "./resolve" export async function connect(config: ConnectionConfig): Promise { @@ -17,6 +19,85 @@ export async function connect(config: ConnectionConfig): Promise { return value.replace(/`/g, "``") } + // How long to wait for the server to acknowledge a cancel before reporting the statement as + // possibly still running. + const CANCEL_WAIT_MS = 30_000 + const STOPPED = Symbol("stopped") + + async function cancelAndClose(operation: any): Promise { + let timer: ReturnType | undefined + const cancelled = await Promise.race([ + operation.cancel().then( + () => true, + () => false, + ), + new Promise((resolve) => { + timer = setTimeout(() => resolve(false), CANCEL_WAIT_MS) + }), + ]).finally(() => clearTimeout(timer)) + operation.close().catch(() => {}) + return cancelled + } + + // `cancelAfterMs` is an opt-in deadline: past it, or when `signal` aborts, the statement is + // cancelled on the warehouse. SQL warehouses ignore the client's server-side `queryTimeout`. + async function runStatement( + query: string, + stmtOptions: Record, + options?: ExecuteOptions, + ): Promise { + const deadlineMs = options?.cancelAfterMs + if (!deadlineMs || deadlineMs <= 0) { + const operation = await session.executeStatement(query, stmtOptions) + const rows = await operation.fetchAll() + await operation.close() + return rows + } + const signal = options?.signal + signal?.throwIfAborted() + let stopped: "deadline" | "aborted" | undefined + let stop: () => void = () => {} + const stopping = new Promise((resolve) => (stop = () => resolve(STOPPED))) + const timer = setTimeout( + () => { + stopped ??= "deadline" + stop() + }, + Math.min(deadlineMs, MAX_TIMER_MS), + ) + const onAbort = () => { + stopped ??= "aborted" + stop() + } + signal?.addEventListener("abort", onAbort, { once: true }) + let operation: any + const work = (async () => { + const started = await session.executeStatement(query, stmtOptions) + // A statement the server accepted only after the stop is cancelled as soon as it exists. + if (stopped) { + await cancelAndClose(started) + return [] as any[] + } + operation = started + return operation.fetchAll() as Promise + })() + let outcome: any[] | typeof STOPPED + try { + outcome = await Promise.race([work, stopping]) + } finally { + clearTimeout(timer) + signal?.removeEventListener("abort", onAbort) + } + if (outcome !== STOPPED) { + await operation.close() + return outcome + } + work.catch(() => {}) + const cancelled = operation ? await cancelAndClose(operation) : false + if (stopped === "aborted") throw signal!.reason + throw new QueryTimeoutError({ engine: "databricks", timeoutMs: deadlineMs, cancelled }) + } + return { async connect() { const DBSQLClient = databricksModule.DBSQLClient ?? databricksModule @@ -55,9 +136,7 @@ export async function connect(config: ConnectionConfig): Promise { if (binds?.length) { stmtOptions.ordinalParameters = binds } - const operation = await session.executeStatement(query, stmtOptions) - const rows = await operation.fetchAll() - await operation.close() + const rows = await runStatement(query, stmtOptions, options) const columns = rows.length > 0 ? Object.keys(rows[0]) : [] const truncated = effectiveLimit > 0 && rows.length > effectiveLimit diff --git a/packages/drivers/src/types.ts b/packages/drivers/src/types.ts index 16fe92a519..00ec89cf00 100644 --- a/packages/drivers/src/types.ts +++ b/packages/drivers/src/types.ts @@ -44,6 +44,10 @@ export interface ExecuteOptions { * `query_timeout_ms`, then `ALTIMATE_SQL_QUERY_TIMEOUT_MS`, then the * 120 s default (see `query-timeout.ts`). */ queryTimeoutMs?: number + /** Cancel the statement on the server once this many milliseconds pass, or when + * `signal` aborts. Opt-in: `executeQuery` fills in `queryTimeoutMs` for every + * call, so only callers that set this get the server-side cancel. Databricks only. */ + cancelAfterMs?: number /** DuckDB only: called once after handing this query to its helper; callback errors do not affect execution. */ onDispatch?: () => void } diff --git a/packages/drivers/test/databricks-deadline.test.ts b/packages/drivers/test/databricks-deadline.test.ts new file mode 100644 index 0000000000..7c50f58d82 --- /dev/null +++ b/packages/drivers/test/databricks-deadline.test.ts @@ -0,0 +1,144 @@ +/** + * Databricks server-side cancel: a caller that sets `cancelAfterMs` gets its statement cancelled + * on the warehouse when the deadline passes or its signal aborts. `queryTimeoutMs`, which every + * `executeQuery` call fills in, does not cancel anything. + */ +import { beforeEach, describe, expect, mock, test } from "bun:test" +import { restoreModuleMocksAfterTests } from "./module-mock" +import { isQueryTimeoutError } from "../src/types" + +await restoreModuleMocksAfterTests("@databricks/sql") + +let fetch: () => Promise +let startDelayMs = 0 +let cancelResult: () => Promise +let cancels = 0 +let closes = 0 + +beforeEach(() => { + fetch = async () => [{ n: 1 }] + startDelayMs = 0 + cancelResult = async () => ({}) + cancels = 0 + closes = 0 +}) + +const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)) + +class FakeClient { + async connect() {} + async openSession() { + return { + executeStatement: async () => { + if (startDelayMs > 0) await sleep(startDelayMs) + return { + fetchAll: () => fetch(), + cancel: async () => { + cancels++ + return cancelResult() + }, + close: async () => { + closes++ + }, + } + }, + } + } +} + +// The driver reads `default` first (the package is CommonJS), so both shapes point at the fake. +mock.module("@databricks/sql", () => ({ default: { DBSQLClient: FakeClient }, DBSQLClient: FakeClient })) + +const { connect } = await import("../src/databricks") + +async function connector() { + const one = await connect({ + type: "databricks", + server_hostname: "h", + http_path: "/sql/1.0/warehouses/w", + access_token: "t", + }) + await one.connect() + return one +} + +const never = () => new Promise(() => {}) + +describe("Databricks server-side cancel", () => { + test("a statement past `cancelAfterMs` is cancelled on the warehouse and reported as a timeout", async () => { + fetch = never + const db = await connector() + const error = await db.execute("SELECT 1", 10, [], { cancelAfterMs: 30 }).catch((e) => e) + expect(isQueryTimeoutError(error)).toBe(true) + expect(error.cancelled).toBe(true) + expect(error.timeoutMs).toBe(30) + expect(error.engine).toBe("databricks") + expect([cancels, closes]).toEqual([1, 1]) + }) + + test("`queryTimeoutMs` alone never cancels: a slow statement still returns its rows", async () => { + fetch = async () => { + await sleep(80) + return [{ n: 3 }] + } + const db = await connector() + const result = await db.execute("SELECT 1", 10, [], { queryTimeoutMs: 30 }) + expect(result.rows).toEqual([[3]]) + expect(cancels).toBe(0) + }) + + test("a statement the server accepts only after the deadline is cancelled once it exists", async () => { + startDelayMs = 80 + const db = await connector() + const error = await db.execute("SELECT 1", 10, [], { cancelAfterMs: 30 }).catch((e) => e) + expect(isQueryTimeoutError(error)).toBe(true) + // Nothing existed to cancel yet, so the error does not claim it was cancelled. + expect(error.cancelled).toBe(false) + expect(cancels).toBe(0) + await sleep(120) + expect([cancels, closes]).toEqual([1, 1]) + }) + + test("an aborted signal cancels the statement and rejects with the abort reason", async () => { + fetch = never + const controller = new AbortController() + const db = await connector() + const pending = db.execute("SELECT 1", 10, [], { cancelAfterMs: 60_000, signal: controller.signal }).catch((e) => e) + await sleep(20) + controller.abort(new Error("session stopped")) + const error = await pending + expect(isQueryTimeoutError(error)).toBe(false) + expect(error.message).toBe("session stopped") + expect(cancels).toBe(1) + }) + + test("when the cancel itself fails, the error says the statement may still be running", async () => { + fetch = never + cancelResult = async () => { + throw new Error("network") + } + const db = await connector() + const error = await db.execute("SELECT 1", 10, [], { cancelAfterMs: 30 }).catch((e) => e) + expect(isQueryTimeoutError(error)).toBe(true) + expect(error.cancelled).toBe(false) + expect(error.message).toContain("it may still be running on the server") + }) + + test("a statement that finishes in time returns its rows and is not cancelled", async () => { + const db = await connector() + const result = await db.execute("SELECT 1", 10, [], { cancelAfterMs: 5_000 }) + expect(result.rows).toEqual([[1]]) + expect([cancels, closes]).toEqual([0, 1]) + }) + + test("a query error inside the deadline is the query's own error, not a timeout", async () => { + fetch = async () => { + throw new Error("[TABLE_OR_VIEW_NOT_FOUND] system.x") + } + const db = await connector() + const error = await db.execute("SELECT 1", 10, [], { cancelAfterMs: 5_000 }).catch((e) => e) + expect(isQueryTimeoutError(error)).toBe(false) + expect(error.message).toContain("TABLE_OR_VIEW_NOT_FOUND") + expect(cancels).toBe(0) + }) +}) diff --git a/packages/opencode/src/altimate/native/finops/databricks-savings.ts b/packages/opencode/src/altimate/native/finops/databricks-savings.ts index 93e711ce38..68b001616b 100644 --- a/packages/opencode/src/altimate/native/finops/databricks-savings.ts +++ b/packages/opencode/src/altimate/native/finops/databricks-savings.ts @@ -9,6 +9,8 @@ import * as core from "@altimateai/altimate-core" import { Flag } from "@opencode-ai/core/flag/flag" import * as Registry from "../connections/registry" +import { isQueryTimeoutError } from "@altimateai/drivers/types" +import type { QueryContext } from "../query-execution" import { resolveFinopsWarehouse } from "./warehouse-resolver" import type { DatabricksSavingsParams, DatabricksSavingsResult, DatabricksSavingsReport } from "../types" @@ -24,6 +26,10 @@ const WINDOW_END_MS = 6 * 3_600_000 /** Rows read per query; a query that fills it says so in the result. */ const ROW_LIMIT = 10_000 +// The whole check stops after this long; a query still running then is cancelled on the warehouse. +// Claude Code runs a plugin's process for ten minutes at most, so the check ends before it is killed. +export const SAVINGS_RUN_LIMIT_MS = 9 * 60_000 +const LIMIT_TEXT = "9-minute limit" /** A SQL warehouse's id in a connection's HTTP path (`/sql/1.0/warehouses/`, or the older `/sql/1.0/endpoints/`). */ const WAREHOUSE_PATH = /\/(?:warehouses|endpoints)\/([0-9a-z]+)\/?$/i @@ -392,6 +398,7 @@ function preferredConnection(): string | undefined { export async function databricksSavings( params: DatabricksSavingsParams, rules: RulesEngine | null = rulesEngine(), + context?: QueryContext, ): Promise { const window = savingsWindow(params.now_ms ?? Date.now()) const failed = (error: string): DatabricksSavingsResult => ({ success: false, window, error }) @@ -422,10 +429,24 @@ export async function databricksSavings( const binds = [window.end] // A query that fills the row limit reads only its first rows; the result names it. const limited: string[] = [] + // Stopping the session cancels the running query and starts no other. + const signal = context?.signal + const stopsAt = Date.now() + SAVINGS_RUN_LIMIT_MS const read = async (what: string, sql: string) => { - const result = await connector.execute(sql, ROW_LIMIT, binds) - if (result.truncated) limited.push(what) - return records(result) + signal?.throwIfAborted() + const left = stopsAt - Date.now() + if (left <= 0) throw new Error(`The check reached its ${LIMIT_TEXT} before the ${what} query.`) + try { + const result = await connector.execute(sql, ROW_LIMIT, binds, { cancelAfterMs: left, signal }) + if (result.truncated) limited.push(what) + return records(result) + } catch (e) { + if (!isQueryTimeoutError(e)) throw e + const outcome = (e as { cancelled?: boolean }).cancelled + ? "and was cancelled on the warehouse" + : "and may still be running on the warehouse" + throw new Error(`The ${what} query ran past the check's ${LIMIT_TEXT} ${outcome}.`) + } } const warehouses = await read("SQL warehouses", DATABRICKS_WAREHOUSE_FACTS_SQL) if (warehouses.length === 0) { @@ -437,6 +458,7 @@ export async function databricksSavings( try { return await read(what, sql) } catch (e) { + if (signal?.aborted) throw e skipped.push({ check, reason: (e instanceof Error ? e.message : String(e)).split("\n")[0]!.slice(0, 300) }) return [] } diff --git a/packages/opencode/src/altimate/native/finops/register.ts b/packages/opencode/src/altimate/native/finops/register.ts index 24aa1d71b5..43d4a263d5 100644 --- a/packages/opencode/src/altimate/native/finops/register.ts +++ b/packages/opencode/src/altimate/native/finops/register.ts @@ -49,8 +49,8 @@ register("finops.warehouse_advice", async (params: WarehouseAdvisorParams): Prom return adviseWarehouse(params) }) -register("finops.databricks_savings", async (params: DatabricksSavingsParams): Promise => { - return databricksSavings(params) +register("finops.databricks_savings", async (params: DatabricksSavingsParams, context): Promise => { + return databricksSavings(params, undefined, context) }) register("finops.unused_resources", async (params: UnusedResourcesParams): Promise => { diff --git a/packages/opencode/src/altimate/tool-cli/daemon.ts b/packages/opencode/src/altimate/tool-cli/daemon.ts index 37d405febc..808df7b4fb 100644 --- a/packages/opencode/src/altimate/tool-cli/daemon.ts +++ b/packages/opencode/src/altimate/tool-cli/daemon.ts @@ -201,7 +201,7 @@ export async function startDaemon(options: DaemonOptions): Promise { const info = (): DaemonInfo => ({ pid: process.pid, directory, version: options.version, startedAt, idleMs: idle, requests }) - const handle = async (request: Request, afterReply: (action: () => void) => void): Promise => { + const handle = async (request: Request, afterReply: (action: () => void) => void, signal: AbortSignal): Promise => { if (request.op === "status") return { id: request.id, status: "ok", title: "Tool daemon", output: "running", metadata: { ...info() } } if (request.op === "stop") { afterReply(() => void stop()) @@ -226,7 +226,7 @@ export async function startDaemon(options: DaemonOptions): Promise { } } dirty = true - const result = await options.runner.run(request.name, request.args ?? {}, { allowWrite: request.allowWrite === true }) + const result = await options.runner.run(request.name, request.args ?? {}, { allowWrite: request.allowWrite === true, signal }) return { id: request.id, ...result } } @@ -240,18 +240,26 @@ export async function startDaemon(options: DaemonOptions): Promise { const server = net.createServer((socket) => { sockets.add(socket) - socket.on("close", () => sockets.delete(socket)) + // A client that goes away (its `tool run` was stopped) stops its runs: tools that honour the + // signal cancel their work, so a warehouse query does not run on for nobody. + const running = new Set() + socket.on("close", () => { + sockets.delete(socket) + for (const controller of running) controller.abort(new Error("The client disconnected.")) + }) socket.on("error", () => {}) lines(socket, (line) => { inflight++ arm() + const controller = new AbortController() + running.add(controller) let afterReply: (() => void) | undefined let stopRequest = false Promise.resolve() .then(() => { const request = JSON.parse(line) as Request stopRequest = request.op === "stop" - return handle(request, (action) => (afterReply = action)) + return handle(request, (action) => (afterReply = action), controller.signal) }) .catch((error): Response => ({ id: "", @@ -261,6 +269,7 @@ export async function startDaemon(options: DaemonOptions): Promise { metadata: {}, })) .then(async (response) => { + running.delete(controller) inflight-- // Release DuckDB/SQLite file locks once idle, before replying, so a client running `dbt build` next finds the file free. // A request that arrives meanwhile waits for the release (see `handle`). Whichever request diff --git a/packages/opencode/src/altimate/tool-cli/stop-signals.ts b/packages/opencode/src/altimate/tool-cli/stop-signals.ts new file mode 100644 index 0000000000..df6cf8e6c3 --- /dev/null +++ b/packages/opencode/src/altimate/tool-cli/stop-signals.ts @@ -0,0 +1,29 @@ +// altimate_change - new file +// How long a stopped run may take to wind down (a warehouse query's cancel waits up to 30 s) before +// the process exits anyway. +const STOP_GRACE_MS = 40_000 +const SIGNAL_EXIT = { SIGINT: 130, SIGTERM: 143 } as const + +/** Runs `work` with a signal that SIGINT or SIGTERM aborts; a second signal, or a run that has not + * wound down within the grace period, exits at once. */ +export async function withStopSignals(work: (signal: AbortSignal) => Promise): Promise { + const controller = new AbortController() + let forced: ReturnType | undefined + const stop = (name: keyof typeof SIGNAL_EXIT) => { + if (controller.signal.aborted) process.exit(SIGNAL_EXIT[name]) + controller.abort(new Error(`Stopped by ${name}.`)) + forced = setTimeout(() => process.exit(SIGNAL_EXIT[name]), STOP_GRACE_MS) + forced.unref?.() + } + const onInt = () => stop("SIGINT") + const onTerm = () => stop("SIGTERM") + process.on("SIGINT", onInt) + process.on("SIGTERM", onTerm) + try { + return await work(controller.signal) + } finally { + process.off("SIGINT", onInt) + process.off("SIGTERM", onTerm) + clearTimeout(forced) + } +} diff --git a/packages/opencode/src/altimate/tools/finops-databricks-savings.ts b/packages/opencode/src/altimate/tools/finops-databricks-savings.ts index 5220b2b0d0..ac92c33c77 100644 --- a/packages/opencode/src/altimate/tools/finops-databricks-savings.ts +++ b/packages/opencode/src/altimate/tools/finops-databricks-savings.ts @@ -18,6 +18,8 @@ type SavingsMetadata = { warehouses?: { id: string; workspace_id?: string | null; name: string; type: string; size: string | null; window_cost: number }[] report?: DatabricksSavingsReport error?: string + /** True when the check did not run because `confirm` was not set. */ + needs_confirmation?: boolean } function dollars(value: number): string { @@ -107,16 +109,37 @@ export function formatSavings(result: DatabricksSavingsResult): string { export const FinopsDatabricksSavingsTool = Tool.define("finops_databricks_savings", { description: - "Check a Databricks workspace for savings with Altimate's rules: SQL warehouses that would cost less as serverless or smaller, idle all-purpose clusters, and scheduled jobs running on all-purpose compute. Read-only queries over seven days of system tables (billing, compute, query history, jobs), covering every workspace those tables show, on the connection's SQL warehouse, which Databricks starts if it is stopped. Returns each finding with its monthly dollar range and evidence, and what each flagged warehouse would cost at other sizes.", + "Check a Databricks workspace for savings with Altimate's rules: SQL warehouses that would cost less as serverless or smaller, idle all-purpose clusters, and scheduled jobs running on all-purpose compute. Read-only queries over seven days of system tables (billing, compute, query history, jobs), covering every workspace those tables show, on the connection's SQL warehouse, which Databricks starts if it is stopped. Returns each finding with its monthly dollar range and evidence, and what each flagged warehouse would cost at other sizes. Ask the user before running it, and pass confirm: true once they agree; without it nothing runs.", parameters: z.object({ warehouse: z .string() .optional() .describe("Databricks connection name. Optional — omit to use the first configured Databricks connection."), + confirm: z + .boolean() + .optional() + .describe( + "Set to true only after the user has agreed to run the check. It runs three queries on their SQL warehouse, which Databricks starts if it is stopped and bills while it runs; the check stops after 9 minutes, cancelling any query still running.", + ), }), - async execute(args) { + async execute(args, ctx) { + if (args.confirm !== true) { + // Asking first is the expected first step, not a failed run. + return { + title: "Databricks Savings: not run, needs the user's go-ahead", + metadata: { success: true, needs_confirmation: true } as SavingsMetadata, + output: + "Not run. The Databricks savings check runs three queries over seven days of system tables on the connection's SQL warehouse, " + + "which Databricks starts if it is stopped and bills while it runs. It stops after 9 minutes, cancelling any query still running. " + + "Ask the user whether to run it; if they agree, call this tool again with confirm: true.", + } + } try { - const result = await Dispatcher.call("finops.databricks_savings", { warehouse: args.warehouse }) + const result = await Dispatcher.call( + "finops.databricks_savings", + { warehouse: args.warehouse }, + { signal: ctx.abort, sessionID: ctx.sessionID }, + ) if (!result.success || !result.report) { const error = result.error ?? "Unknown error" return { diff --git a/packages/opencode/src/cli/cmd/tool.ts b/packages/opencode/src/cli/cmd/tool.ts index 5ba4c2faf6..5dc53fb9c5 100644 --- a/packages/opencode/src/cli/cmd/tool.ts +++ b/packages/opencode/src/cli/cmd/tool.ts @@ -12,6 +12,7 @@ import { cmd } from "./cmd" import { bootstrap } from "../bootstrap" import type { Runner, ToolResult } from "../../altimate/tool-cli/runner" import { resolveWorkspaceTools, workspaceContextSection } from "../../altimate/tool-cli/workspace-tools" +import { withStopSignals } from "../../altimate/tool-cli/stop-signals" /** Reads all of stdin; `readFileSync(0)` works for pipes and for here-docs, which Bun exposes as files. */ function readStdin(): string { @@ -64,7 +65,9 @@ async function inProcess( workspaceTools: boolean, ): Promise { process.chdir(directory) - return withRunner(directory, connections, workspaceTools, (runner) => runner.run(name, toolArgs, { allowWrite })) + return withStopSignals((signal) => + withRunner(directory, connections, workspaceTools, (runner) => runner.run(name, toolArgs, { allowWrite, signal })), + ) } /** Reloads the connections when a source file was created, edited or removed since the last reload. */ diff --git a/packages/opencode/test/altimate/assessment-zero-egress.test.ts b/packages/opencode/test/altimate/assessment-zero-egress.test.ts index 51b99a455b..ff9e0ac83f 100644 --- a/packages/opencode/test/altimate/assessment-zero-egress.test.ts +++ b/packages/opencode/test/altimate/assessment-zero-egress.test.ts @@ -116,7 +116,7 @@ async function run(args: string[], env: Record, setup?: (home: s } const assessment = { ALTIMATE_ASSESSMENT_MODE: "true", ALTIMATE_TELEMETRY_DISABLED: "true" } -const savings = ["tool", "run", "finops_databricks_savings", "--args", "{}", "--json", "--no-daemon"] +const savings = ["tool", "run", "finops_databricks_savings", "--args", '{"confirm":true}', "--json", "--no-daemon"] describe("assessment mode on the packaged CLI", () => { posixOnly("control: without assessment mode, telemetry off still sends the opt-out event to the telemetry endpoint", async () => { @@ -130,6 +130,13 @@ describe("assessment mode on the packaged CLI", () => { expect(JSON.parse(result.stdout).status).toBe("error") }, 120_000) + posixOnly("without the user's go-ahead the check does not run and no host is contacted", async () => { + const unconfirmed = ["tool", "run", "finops_databricks_savings", "--args", "{}", "--json", "--no-daemon"] + const result = await run(unconfirmed, assessment) + expect(result.hosts).toEqual([]) + expect(JSON.parse(result.stdout).output).toContain("call this tool again with confirm: true") + }, 120_000) + posixOnly("startup path (`tool list`): no host is contacted", async () => { const result = await run(["tool", "list"], assessment) expect(result.code).toBe(0) diff --git a/packages/opencode/test/altimate/finops-databricks-savings.test.ts b/packages/opencode/test/altimate/finops-databricks-savings.test.ts index 4d89e483f1..b2da7ef3a6 100644 --- a/packages/opencode/test/altimate/finops-databricks-savings.test.ts +++ b/packages/opencode/test/altimate/finops-databricks-savings.test.ts @@ -6,6 +6,7 @@ import { DATABRICKS_CLUSTER_FACTS_SQL, DATABRICKS_JOB_FACTS_SQL, DATABRICKS_WAREHOUSE_FACTS_SQL, + SAVINGS_RUN_LIMIT_MS, databricksSavings, rulesEngine, savingsWindow, @@ -106,7 +107,7 @@ describe("finops_databricks_savings tool", () => { test("gives Claude the total, each finding with its evidence, and the whole report as metadata", async () => { Dispatcher.register("finops.databricks_savings" as any, async () => REPORT) const tool = await initTool(FinopsDatabricksSavingsTool) - const result = await tool.execute({}, stubCtx()) + const result = await tool.execute({ confirm: true }, stubCtx()) expect(result.title).toBe("Databricks Savings: $53.20–$66.50 a month on 1 resource(s)") expect(result.output).toContain("Total: $53.20–$66.50 a month on 1 resource, each counted once at its largest saving.") @@ -120,13 +121,45 @@ describe("finops_databricks_savings tool", () => { expect(result.metadata.window).toEqual({ start: "2026-11-05", end: "2026-11-12" }) }) + test("without the user's go-ahead nothing runs: the agent is told to ask first", async () => { + let ran = 0 + Dispatcher.register("finops.databricks_savings" as any, async () => { + ran++ + return REPORT + }) + const tool = await initTool(FinopsDatabricksSavingsTool) + for (const args of [{}, { confirm: false }, { warehouse: "dbx" }]) { + const result = await tool.execute(args as any, stubCtx()) + expect(result.title).toBe("Databricks Savings: not run, needs the user's go-ahead") + // Asking first is the expected step, so it is not recorded as a failed run. + expect(result.metadata).toMatchObject({ success: true, needs_confirmation: true }) + expect(result.metadata.report).toBeUndefined() + expect(result.output).toContain("Ask the user whether to run it; if they agree, call this tool again with confirm: true.") + expect(result.output).toContain("runs three queries") + expect(result.output).toContain("It stops after 9 minutes, cancelling any query still running.") + } + expect(ran).toBe(0) + }) + + test("the session's stop signal reaches the check", async () => { + let seen: unknown + Dispatcher.register("finops.databricks_savings" as any, async (_params: unknown, context: unknown) => { + seen = context + return REPORT + }) + const tool = await initTool(FinopsDatabricksSavingsTool) + const ctx = stubCtx() + await tool.execute({ confirm: true }, ctx) + expect(seen).toEqual({ signal: ctx.abort, sessionID: "test" }) + }) + test("no findings says so and still reports how many warehouses were checked", async () => { Dispatcher.register("finops.databricks_savings" as any, async () => ({ ...REPORT, report: { lookback_days: 7, findings: [], total: { monthly_min: 0, monthly_max: 0, resources: 0 }, what_if: [] }, })) const tool = await initTool(FinopsDatabricksSavingsTool) - const result = await tool.execute({}, stubCtx()) + const result = await tool.execute({ confirm: true }, stubCtx()) expect(result.output).toContain("5 SQL warehouses, 0 all-purpose clusters, 0 jobs checked") expect(result.output).toContain("No savings found") }) @@ -138,7 +171,7 @@ describe("finops_databricks_savings tool", () => { error: "[TABLE_OR_VIEW_NOT_FOUND] The table or view `system`.`compute`.`warehouse_events` cannot be found.", })) const tool = await initTool(FinopsDatabricksSavingsTool) - const result = await tool.execute({}, stubCtx()) + const result = await tool.execute({ confirm: true }, stubCtx()) expect(result.title).toBe("Databricks Savings: FAILED") expect(result.metadata.error).toContain("TABLE_OR_VIEW_NOT_FOUND") expect(result.output).not.toContain("No savings found") @@ -256,7 +289,7 @@ describe("cluster and job findings", () => { }, })) const tool = await initTool(FinopsDatabricksSavingsTool) - const result = await tool.execute({}, stubCtx()) + const result = await tool.execute({ confirm: true }, stubCtx()) expect(result.output).toContain("5 SQL warehouses, 3 all-purpose clusters, 4 jobs checked") expect(result.output).toContain("Not checked: Job Running on All-Purpose Compute ([INSUFFICIENT_PERMISSIONS] system.lakeflow.jobs)") expect(result.output).toContain("shared-analytics (cluster) · Idle Cluster Detection · $765–$1,530 a month (critical)") @@ -304,7 +337,7 @@ describe("databricks savings handler over a stubbed connection", () => { const WAREHOUSE_ROW = { workspace_id: "111", warehouse_id: "abc123", warehouse_name: "BI", warehouse_type: "PRO", warehouse_size: "SMALL", total_cost: 27.5 } const JOB_ROW = { workspace_id: "222", job_id: "j1", job_name: "nightly", trigger_type: null, listed_in_job_inventory: true, run_trigger_type: "ONE_TIME", run_count: 9 } const table = (row: Record, truncated = false) => ({ columns: Object.keys(row), rows: [Object.values(row)], truncated }) - let calls: { sql: string; limit: number; binds: unknown[] }[] + let calls: { sql: string; limit: number; binds: unknown[]; timeout?: number; clientTimeout?: number }[] let sent: any beforeEach(() => { @@ -325,8 +358,8 @@ describe("databricks savings handler over a stubbed connection", () => { const connect = (answer: (sql: string) => unknown) => spyOn(Registry, "get").mockResolvedValue({ - execute: async (sql: string, limit: number, binds: unknown[]) => { - calls.push({ sql, limit, binds }) + execute: async (sql: string, limit: number, binds: unknown[], options?: { cancelAfterMs?: number; queryTimeoutMs?: number }) => { + calls.push({ sql, limit, binds, timeout: options?.cancelAfterMs, clientTimeout: options?.queryTimeoutMs }) return answer(sql) }, } as any) @@ -344,6 +377,14 @@ describe("databricks savings handler over a stubbed connection", () => { // Every query takes only the window end: it reads every workspace the system tables show. expect(calls.map((one) => one.sql)).toEqual([DATABRICKS_WAREHOUSE_FACTS_SQL, DATABRICKS_CLUSTER_FACTS_SQL, DATABRICKS_JOB_FACTS_SQL]) for (const one of calls) expect(one.binds).toEqual(["2026-11-18"]) + // Each query gets what is left of the check's 9 minutes, cancelled on the warehouse when it runs + // out; none sets a client-only timeout that would abandon a query still running. + expect(SAVINGS_RUN_LIMIT_MS).toBe(540_000) + for (const one of calls) { + expect(one.timeout).toBeGreaterThan(SAVINGS_RUN_LIMIT_MS - 5_000) + expect(one.timeout).toBeLessThanOrEqual(SAVINGS_RUN_LIMIT_MS) + expect(one.clientTimeout).toBeUndefined() + } expect(sent.lookback_days).toBe(7) expect(sent.warehouses[0]).toMatchObject({ workspace_id: "111", warehouse_id: "abc123", warehouse_size: "SMALL", total_cost: 27.5 }) expect(result.warehouses?.[0]).toMatchObject({ id: "abc123", workspace_id: "111" }) @@ -352,6 +393,67 @@ describe("databricks savings handler over a stubbed connection", () => { expect(result.limited).toEqual([]) }) + test("a query past its limit fails the check with a plain reason, or skips an optional check", async () => { + const timeout = (cancelled: boolean) => Object.assign(new Error("QUERY_TIMEOUT: databricks query exceeded"), { code: "QUERY_TIMEOUT", cancelled }) + let spy = connect((sql) => { + if (sql === DATABRICKS_WAREHOUSE_FACTS_SQL) throw timeout(true) + return table(JOB_ROW) + }) + let result = await databricksSavings({ warehouse: "dbx", now_ms: Date.parse("2026-11-18T10:00:00Z") }, rules) + spy.mockRestore() + expect(result.success).toBe(false) + expect(result.error).toBe("The SQL warehouses query ran past the check's 9-minute limit and was cancelled on the warehouse.") + + calls = [] + spy = connect((sql) => { + if (sql === DATABRICKS_JOB_FACTS_SQL) throw timeout(false) + return sql === DATABRICKS_WAREHOUSE_FACTS_SQL ? table(WAREHOUSE_ROW) : table({ cluster_id: "c1" }) + }) + result = await databricksSavings({ warehouse: "dbx", now_ms: Date.parse("2026-11-18T10:00:00Z") }, rules) + spy.mockRestore() + expect(result.success).toBe(true) + expect(result.skipped).toEqual([ + { check: "Job Running on All-Purpose Compute", reason: "The jobs query ran past the check's 9-minute limit and may still be running on the warehouse." }, + ]) + }) + + test("a query is not started once the check's 9 minutes are used up", async () => { + const realNow = Date.now + let now = realNow() + const clock = spyOn(Date, "now").mockImplementation(() => now) + const spy = connect((sql) => { + // The warehouse query takes the whole limit. + if (sql === DATABRICKS_WAREHOUSE_FACTS_SQL) now += SAVINGS_RUN_LIMIT_MS + return table(WAREHOUSE_ROW) + }) + const result = await databricksSavings({ warehouse: "dbx", now_ms: Date.parse("2026-11-18T10:00:00Z") }, rules) + spy.mockRestore() + clock.mockRestore() + expect(calls.map((one) => one.sql)).toEqual([DATABRICKS_WAREHOUSE_FACTS_SQL]) + expect(result.success).toBe(true) + expect(result.skipped).toEqual([ + { check: "Idle Cluster Detection", reason: "The check reached its 9-minute limit before the all-purpose clusters query." }, + { check: "Job Running on All-Purpose Compute", reason: "The check reached its 9-minute limit before the jobs query." }, + ]) + }) + + test("stopping the session cancels the running query and starts no other", async () => { + const controller = new AbortController() + const spy = connect((sql) => { + if (sql === DATABRICKS_CLUSTER_FACTS_SQL) { + controller.abort(new Error("session stopped")) + throw new Error("session stopped") + } + return table(WAREHOUSE_ROW) + }) + const result = await databricksSavings({ warehouse: "dbx", now_ms: Date.parse("2026-11-18T10:00:00Z") }, rules, { signal: controller.signal }) + spy.mockRestore() + expect(result.success).toBe(false) + expect(result.error).toBe("session stopped") + // The jobs query never started, and the stop is not reported as a skipped check. + expect(calls.map((one) => one.sql)).toEqual([DATABRICKS_WAREHOUSE_FACTS_SQL, DATABRICKS_CLUSTER_FACTS_SQL]) + }) + test("a query that fills the row limit is named, and the tool says so", async () => { const spy = connect((sql) => (sql === DATABRICKS_WAREHOUSE_FACTS_SQL ? table(WAREHOUSE_ROW) : sql === DATABRICKS_JOB_FACTS_SQL ? table(JOB_ROW, true) : table({ cluster_id: "c1" }))) const result = await databricksSavings({ warehouse: "dbx", now_ms: Date.parse("2026-11-18T10:00:00Z") }, rules) diff --git a/packages/opencode/test/altimate/tool-cli/daemon.test.ts b/packages/opencode/test/altimate/tool-cli/daemon.test.ts index 00a4148ae8..0a7dacea55 100644 --- a/packages/opencode/test/altimate/tool-cli/daemon.test.ts +++ b/packages/opencode/test/altimate/tool-cli/daemon.test.ts @@ -5,6 +5,7 @@ import net from "node:net" import { tmpdir } from "node:os" import path from "node:path" import { + connect, socketName, socketPath, runViaDaemon, @@ -86,6 +87,33 @@ describe("daemon round trip", () => { ]) }) + test("a client that disconnects stops its run: the tool's signal aborts", async () => { + const dir = await temp() + const file = path.join(dir, "d.sock") + const started = Promise.withResolvers() + const stopped = Promise.withResolvers() + const runner: Runner = { + ...fakeRunner(), + async run(_name, _args, options) { + started.resolve() + await new Promise((resolve) => + options?.signal?.addEventListener("abort", () => { + stopped.resolve(options.signal!.reason) + resolve() + }), + ) + return { status: "error", title: "stopped", output: "", metadata: {} } + }, + } + const daemon = await startDaemon({ directory: dir, version: "9.9.9", runner, file, idleMs: 60_000 }) + cleanups.push(() => daemon.stop()) + const socket = await connect(file) + socket!.write(JSON.stringify({ id: "1", name: "slow", args: {} }) + "\n") + await started.promise + socket!.destroy() + expect(((await stopped.promise) as Error).message).toBe("The client disconnected.") + }) + test("tool errors and malformed requests come back as error responses", async () => { const { file } = await start() expect((await send(file, { id: "1", name: "fail" }))?.status).toBe("error") diff --git a/packages/opencode/test/altimate/tool-cli/stop-signals.test.ts b/packages/opencode/test/altimate/tool-cli/stop-signals.test.ts new file mode 100644 index 0000000000..93e9bb2c2d --- /dev/null +++ b/packages/opencode/test/altimate/tool-cli/stop-signals.test.ts @@ -0,0 +1,32 @@ +import { afterEach, expect, spyOn, test } from "bun:test" +import { withStopSignals } from "../../../src/altimate/tool-cli/stop-signals" + +const listeners = () => [process.listenerCount("SIGINT"), process.listenerCount("SIGTERM")] +let exits: ReturnType | undefined +afterEach(() => exits?.mockRestore()) + +test("SIGTERM aborts the run's signal, and the handlers are gone afterwards", async () => { + const before = listeners() + const seen = await withStopSignals(async (signal) => { + process.emit("SIGTERM") + return signal.aborted ? (signal.reason as Error).message : "running" + }) + expect(seen).toBe("Stopped by SIGTERM.") + expect(listeners()).toEqual(before) +}) + +test("a second signal exits at once instead of waiting for the run to wind down", async () => { + const codes: unknown[] = [] + exits = spyOn(process, "exit").mockImplementation(((code?: number) => { + codes.push(code) + }) as never) + await withStopSignals(async () => { + process.emit("SIGINT") + process.emit("SIGINT") + }) + expect(codes).toEqual([130]) +}) + +test("a run that is never stopped sees no abort", async () => { + expect(await withStopSignals(async (signal) => signal.aborted)).toBe(false) +}) From a3338f73c2b09111725ef54742045d9a4d18dd72 Mon Sep 17 00:00:00 2001 From: Shreyas Telkar Date: Wed, 18 Nov 2026 13:47:07 -0800 Subject: [PATCH 20/34] fix: the savings check's limit covers connecting and the run's time budget, and only the check cancels - Connecting to the warehouse counts toward the 9-minute limit and yields to a stop. - The limit is the earlier of 9 minutes and an `altimate run --time-budget` cutoff. - A statement that returned in time is closed without waiting, so a slow or failed close cannot fail it. - `tool run` signal handling and the daemon's abort on a client disconnect are removed: they changed every tool, and the 9-minute limit already ends the check before Claude Code would kill the CLI. Co-Authored-By: Claude Opus 5.5 --- .../data-engineering/tools/finops-tools.md | 2 +- packages/drivers/src/databricks.ts | 4 +- .../native/finops/databricks-savings.ts | 47 +++++++++++++++---- .../opencode/src/altimate/tool-cli/daemon.ts | 17 ++----- .../src/altimate/tool-cli/stop-signals.ts | 29 ------------ packages/opencode/src/cli/cmd/tool.ts | 5 +- .../finops-databricks-savings.test.ts | 33 +++++++++++-- .../test/altimate/tool-cli/daemon.test.ts | 28 ----------- .../altimate/tool-cli/stop-signals.test.ts | 32 ------------- 9 files changed, 75 insertions(+), 122 deletions(-) delete mode 100644 packages/opencode/src/altimate/tool-cli/stop-signals.ts delete mode 100644 packages/opencode/test/altimate/tool-cli/stop-signals.test.ts diff --git a/docs/docs/data-engineering/tools/finops-tools.md b/docs/docs/data-engineering/tools/finops-tools.md index 4820ad1bae..2e2f399075 100644 --- a/docs/docs/data-engineering/tools/finops-tools.md +++ b/docs/docs/data-engineering/tools/finops-tools.md @@ -191,7 +191,7 @@ The tool measures each resource's usage; altimate-core decides what qualifies an **What it does not assess.** Databricks Apps spend is not covered, and a serverless SQL warehouse gets only the Over-Sized Warehouse check. A SQL warehouse's cost is its billed usage for the six full days before the window's last day, counting only records Databricks had ingested before that day, spread over the seven-day window as Altimate's pipeline does, so it reads lower than seven days of billing (on one measured workspace, about 7% lower). A cluster's or job's cost is its billed usage for the seven full days before the window's last day. The connection needs the `@databricks/sql` driver installed. -**Asks first, and stops long queries.** The check runs only when called with `confirm: true`; without it, the agent is told to ask you first, since a run can start a stopped warehouse that then bills. It runs three queries and stops after 9 minutes, cancelling any query still running on the warehouse; stopping the session, or the `tool run` process, cancels the running query too. The Claude Code `/savings` command passes `confirm: true`, since typing it is the go-ahead. +**Asks first, and stops long queries.** The check runs only when called with `confirm: true`; without it, the agent is told to ask you first, since a run can start a stopped warehouse that then bills. It runs three queries and stops after 9 minutes, connecting included, cancelling any query still running on the warehouse (or sooner, at an `altimate run --time-budget` cutoff); stopping the session cancels the running query too. The Claude Code `/savings` command passes `confirm: true`, since typing it is the go-ahead. **Assessment mode.** Set `ALTIMATE_ASSESSMENT_MODE=true` to keep the run from contacting anything but your warehouse. It sends no telemetry (not even the one-time opt-out event), treats you as signed out of Altimate (no workspace or project-binding lookups), fetches no remote config and installs no plugin dependencies, and downloads neither the model catalog (models.dev) nor ripgrep (github.com). Any value but empty, `0` or `false` turns it on, and a tool daemon started without it is not reused. It is measured for `tool run`; other commands (the update check, language-server downloads) are not covered. Core's compiled analysis is not audited for network use. Without it, `ALTIMATE_TELEMETRY_DISABLED=true` alone still lets the CLI send that opt-out event and fetch those. The Claude Code `/savings` command sets it for you. Tool results shown in a Claude Code conversation reach its model provider; assessment mode does not change that. diff --git a/packages/drivers/src/databricks.ts b/packages/drivers/src/databricks.ts index 15652478f8..baa47c62a1 100644 --- a/packages/drivers/src/databricks.ts +++ b/packages/drivers/src/databricks.ts @@ -89,7 +89,9 @@ export async function connect(config: ConnectionConfig): Promise { signal?.removeEventListener("abort", onAbort) } if (outcome !== STOPPED) { - await operation.close() + // Closing frees the server's result; a slow or failed close must not hold up or fail a query + // that already returned. + operation.close().catch(() => {}) return outcome } work.catch(() => {}) diff --git a/packages/opencode/src/altimate/native/finops/databricks-savings.ts b/packages/opencode/src/altimate/native/finops/databricks-savings.ts index 68b001616b..8888f8e9c3 100644 --- a/packages/opencode/src/altimate/native/finops/databricks-savings.ts +++ b/packages/opencode/src/altimate/native/finops/databricks-savings.ts @@ -11,6 +11,7 @@ import { Flag } from "@opencode-ai/core/flag/flag" import * as Registry from "../connections/registry" import { isQueryTimeoutError } from "@altimateai/drivers/types" import type { QueryContext } from "../query-execution" +import { RunBudget } from "../../run-budget" import { resolveFinopsWarehouse } from "./warehouse-resolver" import type { DatabricksSavingsParams, DatabricksSavingsResult, DatabricksSavingsReport } from "../types" @@ -26,10 +27,10 @@ const WINDOW_END_MS = 6 * 3_600_000 /** Rows read per query; a query that fills it says so in the result. */ const ROW_LIMIT = 10_000 -// The whole check stops after this long; a query still running then is cancelled on the warehouse. -// Claude Code runs a plugin's process for ten minutes at most, so the check ends before it is killed. +// The whole check, connecting included, stops after this long; a query still running then is +// cancelled on the warehouse. Claude Code runs a plugin's process for ten minutes at most, so the +// check ends before it is killed. export const SAVINGS_RUN_LIMIT_MS = 9 * 60_000 -const LIMIT_TEXT = "9-minute limit" /** A SQL warehouse's id in a connection's HTTP path (`/sql/1.0/warehouses/`, or the older `/sql/1.0/endpoints/`). */ const WAREHOUSE_PATH = /\/(?:warehouses|endpoints)\/([0-9a-z]+)\/?$/i @@ -424,18 +425,46 @@ export async function databricksSavings( return failed("This altimate-code build's altimate-core has no Databricks savings rules. Update altimate-code, then try again.") } + // Stopping the session cancels the running query and starts no other. The check stops at its own + // limit, or earlier at the session's query cutoff (`altimate run --time-budget`). + const signal = context?.signal + const ownLimitAt = Date.now() + SAVINGS_RUN_LIMIT_MS + const sessionCutoff = context?.sessionID ? RunBudget.queryDeadline(context.sessionID) : undefined + const stopsAt = Math.min(ownLimitAt, sessionCutoff ?? Infinity) + const limitName = stopsAt < ownLimitAt ? "the run's time budget" : "the check's 9-minute limit" + + // Connecting can wait on a stopped warehouse: it counts toward the limit and yields to a stop. + const beforeStop = async (work: Promise, what: string): Promise => { + signal?.throwIfAborted() + let timer: ReturnType | undefined + let onAbort: (() => void) | undefined + try { + return await Promise.race([ + work, + new Promise((_, reject) => { + timer = setTimeout(() => reject(new Error(`${what} took longer than ${limitName}.`)), Math.max(0, stopsAt - Date.now())) + }), + new Promise((_, reject) => { + onAbort = () => reject(signal!.reason) + signal?.addEventListener("abort", onAbort, { once: true }) + }), + ]) + } finally { + clearTimeout(timer) + if (onAbort) signal?.removeEventListener("abort", onAbort) + work.catch(() => {}) + } + } + try { - const connector = await Registry.get(resolved.warehouse) + const connector = await beforeStop(Registry.get(resolved.warehouse), "Connecting to the warehouse") const binds = [window.end] // A query that fills the row limit reads only its first rows; the result names it. const limited: string[] = [] - // Stopping the session cancels the running query and starts no other. - const signal = context?.signal - const stopsAt = Date.now() + SAVINGS_RUN_LIMIT_MS const read = async (what: string, sql: string) => { signal?.throwIfAborted() const left = stopsAt - Date.now() - if (left <= 0) throw new Error(`The check reached its ${LIMIT_TEXT} before the ${what} query.`) + if (left <= 0) throw new Error(`The ${what} query was not started: it would run past ${limitName}.`) try { const result = await connector.execute(sql, ROW_LIMIT, binds, { cancelAfterMs: left, signal }) if (result.truncated) limited.push(what) @@ -445,7 +474,7 @@ export async function databricksSavings( const outcome = (e as { cancelled?: boolean }).cancelled ? "and was cancelled on the warehouse" : "and may still be running on the warehouse" - throw new Error(`The ${what} query ran past the check's ${LIMIT_TEXT} ${outcome}.`) + throw new Error(`The ${what} query ran past ${limitName} ${outcome}.`) } } const warehouses = await read("SQL warehouses", DATABRICKS_WAREHOUSE_FACTS_SQL) diff --git a/packages/opencode/src/altimate/tool-cli/daemon.ts b/packages/opencode/src/altimate/tool-cli/daemon.ts index 808df7b4fb..37d405febc 100644 --- a/packages/opencode/src/altimate/tool-cli/daemon.ts +++ b/packages/opencode/src/altimate/tool-cli/daemon.ts @@ -201,7 +201,7 @@ export async function startDaemon(options: DaemonOptions): Promise { const info = (): DaemonInfo => ({ pid: process.pid, directory, version: options.version, startedAt, idleMs: idle, requests }) - const handle = async (request: Request, afterReply: (action: () => void) => void, signal: AbortSignal): Promise => { + const handle = async (request: Request, afterReply: (action: () => void) => void): Promise => { if (request.op === "status") return { id: request.id, status: "ok", title: "Tool daemon", output: "running", metadata: { ...info() } } if (request.op === "stop") { afterReply(() => void stop()) @@ -226,7 +226,7 @@ export async function startDaemon(options: DaemonOptions): Promise { } } dirty = true - const result = await options.runner.run(request.name, request.args ?? {}, { allowWrite: request.allowWrite === true, signal }) + const result = await options.runner.run(request.name, request.args ?? {}, { allowWrite: request.allowWrite === true }) return { id: request.id, ...result } } @@ -240,26 +240,18 @@ export async function startDaemon(options: DaemonOptions): Promise { const server = net.createServer((socket) => { sockets.add(socket) - // A client that goes away (its `tool run` was stopped) stops its runs: tools that honour the - // signal cancel their work, so a warehouse query does not run on for nobody. - const running = new Set() - socket.on("close", () => { - sockets.delete(socket) - for (const controller of running) controller.abort(new Error("The client disconnected.")) - }) + socket.on("close", () => sockets.delete(socket)) socket.on("error", () => {}) lines(socket, (line) => { inflight++ arm() - const controller = new AbortController() - running.add(controller) let afterReply: (() => void) | undefined let stopRequest = false Promise.resolve() .then(() => { const request = JSON.parse(line) as Request stopRequest = request.op === "stop" - return handle(request, (action) => (afterReply = action), controller.signal) + return handle(request, (action) => (afterReply = action)) }) .catch((error): Response => ({ id: "", @@ -269,7 +261,6 @@ export async function startDaemon(options: DaemonOptions): Promise { metadata: {}, })) .then(async (response) => { - running.delete(controller) inflight-- // Release DuckDB/SQLite file locks once idle, before replying, so a client running `dbt build` next finds the file free. // A request that arrives meanwhile waits for the release (see `handle`). Whichever request diff --git a/packages/opencode/src/altimate/tool-cli/stop-signals.ts b/packages/opencode/src/altimate/tool-cli/stop-signals.ts deleted file mode 100644 index df6cf8e6c3..0000000000 --- a/packages/opencode/src/altimate/tool-cli/stop-signals.ts +++ /dev/null @@ -1,29 +0,0 @@ -// altimate_change - new file -// How long a stopped run may take to wind down (a warehouse query's cancel waits up to 30 s) before -// the process exits anyway. -const STOP_GRACE_MS = 40_000 -const SIGNAL_EXIT = { SIGINT: 130, SIGTERM: 143 } as const - -/** Runs `work` with a signal that SIGINT or SIGTERM aborts; a second signal, or a run that has not - * wound down within the grace period, exits at once. */ -export async function withStopSignals(work: (signal: AbortSignal) => Promise): Promise { - const controller = new AbortController() - let forced: ReturnType | undefined - const stop = (name: keyof typeof SIGNAL_EXIT) => { - if (controller.signal.aborted) process.exit(SIGNAL_EXIT[name]) - controller.abort(new Error(`Stopped by ${name}.`)) - forced = setTimeout(() => process.exit(SIGNAL_EXIT[name]), STOP_GRACE_MS) - forced.unref?.() - } - const onInt = () => stop("SIGINT") - const onTerm = () => stop("SIGTERM") - process.on("SIGINT", onInt) - process.on("SIGTERM", onTerm) - try { - return await work(controller.signal) - } finally { - process.off("SIGINT", onInt) - process.off("SIGTERM", onTerm) - clearTimeout(forced) - } -} diff --git a/packages/opencode/src/cli/cmd/tool.ts b/packages/opencode/src/cli/cmd/tool.ts index 5dc53fb9c5..5ba4c2faf6 100644 --- a/packages/opencode/src/cli/cmd/tool.ts +++ b/packages/opencode/src/cli/cmd/tool.ts @@ -12,7 +12,6 @@ import { cmd } from "./cmd" import { bootstrap } from "../bootstrap" import type { Runner, ToolResult } from "../../altimate/tool-cli/runner" import { resolveWorkspaceTools, workspaceContextSection } from "../../altimate/tool-cli/workspace-tools" -import { withStopSignals } from "../../altimate/tool-cli/stop-signals" /** Reads all of stdin; `readFileSync(0)` works for pipes and for here-docs, which Bun exposes as files. */ function readStdin(): string { @@ -65,9 +64,7 @@ async function inProcess( workspaceTools: boolean, ): Promise { process.chdir(directory) - return withStopSignals((signal) => - withRunner(directory, connections, workspaceTools, (runner) => runner.run(name, toolArgs, { allowWrite, signal })), - ) + return withRunner(directory, connections, workspaceTools, (runner) => runner.run(name, toolArgs, { allowWrite })) } /** Reloads the connections when a source file was created, edited or removed since the last reload. */ diff --git a/packages/opencode/test/altimate/finops-databricks-savings.test.ts b/packages/opencode/test/altimate/finops-databricks-savings.test.ts index b2da7ef3a6..211efcec48 100644 --- a/packages/opencode/test/altimate/finops-databricks-savings.test.ts +++ b/packages/opencode/test/altimate/finops-databricks-savings.test.ts @@ -2,6 +2,7 @@ import { afterEach, beforeEach, describe, expect, spyOn, test } from "bun:test" import { initTool } from "./tool-fixture" import * as Dispatcher from "../../src/altimate/native/dispatcher" import * as Registry from "../../src/altimate/native/connections/registry" +import { RunBudget } from "../../src/altimate/run-budget" import { DATABRICKS_CLUSTER_FACTS_SQL, DATABRICKS_JOB_FACTS_SQL, @@ -337,7 +338,7 @@ describe("databricks savings handler over a stubbed connection", () => { const WAREHOUSE_ROW = { workspace_id: "111", warehouse_id: "abc123", warehouse_name: "BI", warehouse_type: "PRO", warehouse_size: "SMALL", total_cost: 27.5 } const JOB_ROW = { workspace_id: "222", job_id: "j1", job_name: "nightly", trigger_type: null, listed_in_job_inventory: true, run_trigger_type: "ONE_TIME", run_count: 9 } const table = (row: Record, truncated = false) => ({ columns: Object.keys(row), rows: [Object.values(row)], truncated }) - let calls: { sql: string; limit: number; binds: unknown[]; timeout?: number; clientTimeout?: number }[] + let calls: { sql: string; limit: number; binds: unknown[]; timeout?: number; clientTimeout?: number; signal?: AbortSignal }[] let sent: any beforeEach(() => { @@ -358,8 +359,8 @@ describe("databricks savings handler over a stubbed connection", () => { const connect = (answer: (sql: string) => unknown) => spyOn(Registry, "get").mockResolvedValue({ - execute: async (sql: string, limit: number, binds: unknown[], options?: { cancelAfterMs?: number; queryTimeoutMs?: number }) => { - calls.push({ sql, limit, binds, timeout: options?.cancelAfterMs, clientTimeout: options?.queryTimeoutMs }) + execute: async (sql: string, limit: number, binds: unknown[], options?: { cancelAfterMs?: number; queryTimeoutMs?: number; signal?: AbortSignal }) => { + calls.push({ sql, limit, binds, timeout: options?.cancelAfterMs, clientTimeout: options?.queryTimeoutMs, signal: options?.signal }) return answer(sql) }, } as any) @@ -432,11 +433,31 @@ describe("databricks savings handler over a stubbed connection", () => { expect(calls.map((one) => one.sql)).toEqual([DATABRICKS_WAREHOUSE_FACTS_SQL]) expect(result.success).toBe(true) expect(result.skipped).toEqual([ - { check: "Idle Cluster Detection", reason: "The check reached its 9-minute limit before the all-purpose clusters query." }, - { check: "Job Running on All-Purpose Compute", reason: "The check reached its 9-minute limit before the jobs query." }, + { check: "Idle Cluster Detection", reason: "The all-purpose clusters query was not started: it would run past the check's 9-minute limit." }, + { check: "Job Running on All-Purpose Compute", reason: "The jobs query was not started: it would run past the check's 9-minute limit." }, ]) }) + test("connecting counts toward the limit: a connection that never comes up ends the check in time", async () => { + // A 30 ms session budget stands in for the last moments of the 9 minutes. + const budget = spyOn(RunBudget, "queryDeadline").mockReturnValue(Date.now() + 30) + const get = spyOn(Registry, "get").mockReturnValue(new Promise(() => {}) as any) + const result = await databricksSavings({ warehouse: "dbx", now_ms: Date.parse("2026-11-18T10:00:00Z") }, rules, { sessionID: "s1" }) + get.mockRestore() + budget.mockRestore() + expect(result.success).toBe(false) + expect(result.error).toBe("Connecting to the warehouse took longer than the run's time budget.") + }) + + test("a session time budget shorter than 9 minutes is the limit", async () => { + const budget = spyOn(RunBudget, "queryDeadline").mockReturnValue(Date.now() + 60_000) + const spy = connect((sql) => (sql === DATABRICKS_WAREHOUSE_FACTS_SQL ? table(WAREHOUSE_ROW) : table(JOB_ROW))) + await databricksSavings({ warehouse: "dbx", now_ms: Date.parse("2026-11-18T10:00:00Z") }, rules, { sessionID: "s1" }) + spy.mockRestore() + budget.mockRestore() + for (const one of calls) expect(one.timeout).toBeLessThanOrEqual(60_000) + }) + test("stopping the session cancels the running query and starts no other", async () => { const controller = new AbortController() const spy = connect((sql) => { @@ -450,6 +471,8 @@ describe("databricks savings handler over a stubbed connection", () => { spy.mockRestore() expect(result.success).toBe(false) expect(result.error).toBe("session stopped") + // Each query got the session's signal, so the driver cancels it on the warehouse. + for (const one of calls) expect(one.signal).toBe(controller.signal) // The jobs query never started, and the stop is not reported as a skipped check. expect(calls.map((one) => one.sql)).toEqual([DATABRICKS_WAREHOUSE_FACTS_SQL, DATABRICKS_CLUSTER_FACTS_SQL]) }) diff --git a/packages/opencode/test/altimate/tool-cli/daemon.test.ts b/packages/opencode/test/altimate/tool-cli/daemon.test.ts index 0a7dacea55..00a4148ae8 100644 --- a/packages/opencode/test/altimate/tool-cli/daemon.test.ts +++ b/packages/opencode/test/altimate/tool-cli/daemon.test.ts @@ -5,7 +5,6 @@ import net from "node:net" import { tmpdir } from "node:os" import path from "node:path" import { - connect, socketName, socketPath, runViaDaemon, @@ -87,33 +86,6 @@ describe("daemon round trip", () => { ]) }) - test("a client that disconnects stops its run: the tool's signal aborts", async () => { - const dir = await temp() - const file = path.join(dir, "d.sock") - const started = Promise.withResolvers() - const stopped = Promise.withResolvers() - const runner: Runner = { - ...fakeRunner(), - async run(_name, _args, options) { - started.resolve() - await new Promise((resolve) => - options?.signal?.addEventListener("abort", () => { - stopped.resolve(options.signal!.reason) - resolve() - }), - ) - return { status: "error", title: "stopped", output: "", metadata: {} } - }, - } - const daemon = await startDaemon({ directory: dir, version: "9.9.9", runner, file, idleMs: 60_000 }) - cleanups.push(() => daemon.stop()) - const socket = await connect(file) - socket!.write(JSON.stringify({ id: "1", name: "slow", args: {} }) + "\n") - await started.promise - socket!.destroy() - expect(((await stopped.promise) as Error).message).toBe("The client disconnected.") - }) - test("tool errors and malformed requests come back as error responses", async () => { const { file } = await start() expect((await send(file, { id: "1", name: "fail" }))?.status).toBe("error") diff --git a/packages/opencode/test/altimate/tool-cli/stop-signals.test.ts b/packages/opencode/test/altimate/tool-cli/stop-signals.test.ts deleted file mode 100644 index 93e9bb2c2d..0000000000 --- a/packages/opencode/test/altimate/tool-cli/stop-signals.test.ts +++ /dev/null @@ -1,32 +0,0 @@ -import { afterEach, expect, spyOn, test } from "bun:test" -import { withStopSignals } from "../../../src/altimate/tool-cli/stop-signals" - -const listeners = () => [process.listenerCount("SIGINT"), process.listenerCount("SIGTERM")] -let exits: ReturnType | undefined -afterEach(() => exits?.mockRestore()) - -test("SIGTERM aborts the run's signal, and the handlers are gone afterwards", async () => { - const before = listeners() - const seen = await withStopSignals(async (signal) => { - process.emit("SIGTERM") - return signal.aborted ? (signal.reason as Error).message : "running" - }) - expect(seen).toBe("Stopped by SIGTERM.") - expect(listeners()).toEqual(before) -}) - -test("a second signal exits at once instead of waiting for the run to wind down", async () => { - const codes: unknown[] = [] - exits = spyOn(process, "exit").mockImplementation(((code?: number) => { - codes.push(code) - }) as never) - await withStopSignals(async () => { - process.emit("SIGINT") - process.emit("SIGINT") - }) - expect(codes).toEqual([130]) -}) - -test("a run that is never stopped sees no abort", async () => { - expect(await withStopSignals(async (signal) => signal.aborted)).toBe(false) -}) From a80b2f16adf1d902237f00e4c9cdfe98cc260f88 Mon Sep 17 00:00:00 2001 From: Shreyas Telkar Date: Wed, 18 Nov 2026 14:19:05 -0800 Subject: [PATCH 21/34] fix: the jobs savings check finds jobs through the compute their task runs used Databricks leaves the job id empty on all-purpose billing, so the check never fired. It now reads the task run timeline's compute ids, one row per job, task, cluster and trigger, with the cluster's own billing to price the runs. Jobs since deleted are left out, and a job counts once in the number checked. Co-Authored-By: Claude Opus 5.5 --- .../data-engineering/tools/finops-tools.md | 4 +- .../native/finops/databricks-savings.ts | 166 +++++++++++------- .../tools/finops-databricks-savings.ts | 13 +- .../finops-databricks-savings.test.ts | 101 ++++++++--- 4 files changed, 193 insertions(+), 91 deletions(-) diff --git a/docs/docs/data-engineering/tools/finops-tools.md b/docs/docs/data-engineering/tools/finops-tools.md index 2e2f399075..b984c10c7e 100644 --- a/docs/docs/data-engineering/tools/finops-tools.md +++ b/docs/docs/data-engineering/tools/finops-tools.md @@ -185,11 +185,11 @@ Check a Databricks workspace for savings with Altimate's rules, on a Databricks - **Migrate Classic/Pro to Serverless:** a Classic or Pro warehouse that bills for time it sits idle. - **Over-Sized Warehouse:** a warehouse whose queries never use the cores it pays for. - **Idle Cluster Detection:** an all-purpose cluster with idle workers and auto-termination off or an hour or more. -- **Job Running on All-Purpose Compute:** a scheduled job that would bill at the jobs-compute rate on a job cluster. +- **Job Running on All-Purpose Compute:** a job with at least one scheduled run in the seven days, or at least 24 hours of continuous running in one task, on an all-purpose cluster. Its DBUs would bill at the jobs-compute rate on a job cluster. The tool measures each resource's usage; altimate-core decides what qualifies and what it saves. Prices are list prices. Each resource is counted once in the total, and each flagged warehouse also gets what it would cost at every smaller size and as serverless. If the connection cannot read the cluster or job tables, that check is named as not checked and the warehouse checks still report; the warehouse tables themselves are required. -**What it does not assess.** Databricks Apps spend is not covered, and a serverless SQL warehouse gets only the Over-Sized Warehouse check. A SQL warehouse's cost is its billed usage for the six full days before the window's last day, counting only records Databricks had ingested before that day, spread over the seven-day window as Altimate's pipeline does, so it reads lower than seven days of billing (on one measured workspace, about 7% lower). A cluster's or job's cost is its billed usage for the seven full days before the window's last day. The connection needs the `@databricks/sql` driver installed. +**What it does not assess.** Databricks Apps spend is not covered, and a serverless SQL warehouse gets only the Over-Sized Warehouse check. A SQL warehouse's cost is its billed usage for the six full days before the window's last day, counting only records Databricks had ingested before that day, spread over the seven-day window as Altimate's pipeline does, so it reads lower than seven days of billing (on one measured workspace, about 7% lower). A cluster's cost is its billed usage for the seven full days before the window's last day. A job's cost is estimated, because Databricks billing does not name the job on all-purpose compute: its task hours from the job timeline times the DBUs per hour its cluster billed. Jobs sharing a cluster never add up to more DBUs than the cluster billed. The connection needs the `@databricks/sql` driver installed. **Asks first, and stops long queries.** The check runs only when called with `confirm: true`; without it, the agent is told to ask you first, since a run can start a stopped warehouse that then bills. It runs three queries and stops after 9 minutes, connecting included, cancelling any query still running on the warehouse (or sooner, at an `altimate run --time-budget` cutoff); stopping the session cancels the running query too. The Claude Code `/savings` command passes `confirm: true`, since typing it is the go-ahead. diff --git a/packages/opencode/src/altimate/native/finops/databricks-savings.ts b/packages/opencode/src/altimate/native/finops/databricks-savings.ts index 8888f8e9c3..1fdf386e60 100644 --- a/packages/opencode/src/altimate/native/finops/databricks-savings.ts +++ b/packages/opencode/src/altimate/native/finops/databricks-savings.ts @@ -166,7 +166,8 @@ const LIST_PRICE_JOIN = `LEFT JOIN system.billing.list_prices lp * Per all-purpose cluster (created from the UI or API, not deleted) of every workspace the * connection's system tables cover: worker node minutes, active days, idle worker minutes (CPU * under 5%, CPU wait under 5%, under 10 MB of network a minute), CPU averages, and the - * all-purpose cost billed outside job runs. Seven full days before the window end, the parameter. + * all-purpose cost it billed (Databricks does not mark job runs on these billing rows, so their + * time is included). Seven full days before the window end, the parameter. */ export const DATABRICKS_CLUSTER_FACTS_SQL = ` WITH p AS ( @@ -218,73 +219,108 @@ ORDER BY COALESCE(bill.total_cost, 0.0) DESC, inv.cluster_name ` /** - * Per job of every workspace the connection's system tables cover that billed in the window: its - * cost on every product and on all-purpose compute, its share of the all-purpose cluster it used - * most, its runs and trigger, and its workspace's jobs-compute rate. Job ids repeat across - * workspaces, so every key pairs the workspace with the id. Seven full days before the window end, - * the parameter. + * Per job task of every workspace the connection's system tables cover that ran on an all-purpose + * cluster (created from the UI or API, not deleted) in the window, one row per job, task, cluster + * and trigger: its runs and hours, what started them, and the cluster's own all-purpose billing and + * the workspace's jobs-compute rate to price them. The task timeline names the compute a run used + * (billing does not name the job on all-purpose compute). It splits a run into contiguous rows cut + * at each hour, so a run's hours are the sum of its rows, credited to the cluster it spent most of + * them on. Deleted jobs are left out. Job ids repeat across workspaces, so every key pairs the + * workspace with the id. Seven full days before the window end, the parameter. */ export const DATABRICKS_JOB_FACTS_SQL = ` WITH p AS ( SELECT CAST(? AS DATE) AS we_date ), +clusters AS ( + SELECT workspace_id, cluster_id, cluster_name FROM ( + SELECT workspace_id, cluster_id, cluster_name, cluster_source, delete_time + FROM system.compute.clusters + QUALIFY ROW_NUMBER() OVER (PARTITION BY workspace_id, cluster_id ORDER BY change_time DESC) = 1 + ) + WHERE delete_time IS NULL AND cluster_source IN ('UI', 'API') +), +timeline AS ( + SELECT t.workspace_id, t.job_id, t.task_key, COALESCE(NULLIF(t.job_run_id, ''), t.run_id) AS job_run_id, + t.run_id, t.period_start_time, t.period_end_time, explode(t.compute_ids) AS compute_id + FROM system.lakeflow.job_task_run_timeline t CROSS JOIN p + WHERE t.period_start_time >= CAST(DATE_SUB(p.we_date, 7) AS TIMESTAMP) AND t.period_start_time < CAST(p.we_date AS TIMESTAMP) +), +timeline_row AS ( + SELECT tl.workspace_id, tl.job_id, tl.task_key, tl.job_run_id, tl.run_id, tl.period_start_time, + MIN(c.cluster_id) AS cluster_id, + MAX(timestampdiff(SECOND, tl.period_start_time, tl.period_end_time)) AS seconds + FROM timeline tl + JOIN clusters c ON c.workspace_id = tl.workspace_id AND c.cluster_id = tl.compute_id + GROUP BY tl.workspace_id, tl.job_id, tl.task_key, tl.job_run_id, tl.run_id, tl.period_start_time +), +run_on_cluster AS ( + SELECT workspace_id, job_id, task_key, job_run_id, cluster_id, SUM(seconds) AS seconds + FROM timeline_row + GROUP BY workspace_id, job_id, task_key, job_run_id, cluster_id +), +task_run AS ( + SELECT workspace_id, job_id, task_key, job_run_id, + MIN_BY(cluster_id, struct(-seconds, cluster_id)) AS cluster_id, + SUM(seconds) / 3600.0 AS hours + FROM run_on_cluster + GROUP BY workspace_id, job_id, task_key, job_run_id +), +run_trigger AS ( + SELECT r.workspace_id, r.run_id, MAX(r.trigger_type) AS trigger_type + FROM system.lakeflow.job_run_timeline r CROSS JOIN p + WHERE r.period_start_time >= CAST(DATE_SUB(p.we_date, 7) AS TIMESTAMP) AND r.period_start_time < CAST(p.we_date AS TIMESTAMP) + GROUP BY r.workspace_id, r.run_id +), +task AS ( + SELECT tr.workspace_id, tr.job_id, tr.task_key, tr.cluster_id, rt.trigger_type, + COUNT(*) AS task_runs, SUM(tr.hours) AS task_hours + FROM task_run tr + LEFT JOIN run_trigger rt ON rt.workspace_id = tr.workspace_id AND rt.run_id = tr.job_run_id + GROUP BY tr.workspace_id, tr.job_id, tr.task_key, tr.cluster_id, rt.trigger_type +), bill AS ( - SELECT u.workspace_id, u.usage_metadata.job_id AS job_id, u.usage_metadata.cluster_id AS cluster_id, - u.billing_origin_product AS product, u.usage_quantity AS dbus, u.usage_quantity * lp.pricing.default AS cost + SELECT u.workspace_id, u.usage_metadata.cluster_id AS cluster_id, u.billing_origin_product AS product, + u.usage_start_time, u.usage_end_time, u.usage_quantity AS dbus, u.usage_quantity * lp.pricing.default AS cost FROM system.billing.usage u CROSS JOIN p ${LIST_PRICE_JOIN} - WHERE u.usage_date >= DATE_SUB(p.we_date, 7) AND u.usage_date < p.we_date -), -job_bill AS ( - SELECT workspace_id, job_id, - SUM(COALESCE(cost, 0)) AS job_total_cost, - SUM(CASE WHEN product = 'ALL_PURPOSE' THEN COALESCE(cost, 0) ELSE 0 END) AS job_all_purpose_cost, - SUM(CASE WHEN product = 'ALL_PURPOSE' THEN dbus ELSE 0 END) AS job_all_purpose_dbus - FROM bill WHERE job_id IS NOT NULL AND job_id <> '' GROUP BY workspace_id, job_id -), -job_cluster AS ( - SELECT workspace_id, job_id, cluster_id, SUM(COALESCE(cost, 0)) AS cost - FROM bill WHERE product = 'ALL_PURPOSE' AND job_id IS NOT NULL AND job_id <> '' AND cluster_id IS NOT NULL - GROUP BY workspace_id, job_id, cluster_id + WHERE u.billing_origin_product IN ('ALL_PURPOSE', 'JOBS') + AND u.usage_date >= DATE_SUB(p.we_date, 7) AND u.usage_date < p.we_date ), -main_cluster AS ( - SELECT workspace_id, job_id, cluster_id, cost FROM job_cluster - QUALIFY ROW_NUMBER() OVER (PARTITION BY workspace_id, job_id ORDER BY cost DESC, cluster_id) = 1 +billed_period AS ( + SELECT workspace_id, cluster_id, usage_start_time, usage_end_time, + SUM(COALESCE(cost, 0)) AS cost, SUM(dbus) AS dbus, + timestampdiff(SECOND, usage_start_time, usage_end_time) AS seconds + FROM bill WHERE product = 'ALL_PURPOSE' AND cluster_id IS NOT NULL + GROUP BY workspace_id, cluster_id, usage_start_time, usage_end_time ), -cluster_total AS ( - SELECT workspace_id, cluster_id, SUM(COALESCE(cost, 0)) AS cost - FROM bill WHERE product = 'ALL_PURPOSE' AND cluster_id IS NOT NULL GROUP BY workspace_id, cluster_id +cluster_bill AS ( + SELECT workspace_id, cluster_id, SUM(cost) AS cost, SUM(dbus) AS dbus, SUM(seconds) / 3600.0 AS billed_hours + FROM billed_period GROUP BY workspace_id, cluster_id ), jobs_rate AS ( SELECT workspace_id, CASE WHEN SUM(dbus) > 0 THEN SUM(COALESCE(cost, 0)) / SUM(dbus) END AS rate FROM bill WHERE product = 'JOBS' GROUP BY workspace_id ), -runs AS ( - SELECT r.workspace_id, r.job_id, COUNT(DISTINCT r.run_id) AS run_count, MAX(r.trigger_type) AS run_trigger - FROM system.lakeflow.job_run_timeline r CROSS JOIN p - WHERE r.period_start_time >= CAST(DATE_SUB(p.we_date, 7) AS TIMESTAMP) AND r.period_start_time < CAST(p.we_date AS TIMESTAMP) - GROUP BY r.workspace_id, r.job_id -), inv AS ( - SELECT workspace_id, job_id, name, trigger_type FROM system.lakeflow.jobs + SELECT workspace_id, job_id, name, delete_time FROM system.lakeflow.jobs QUALIFY ROW_NUMBER() OVER (PARTITION BY workspace_id, job_id ORDER BY change_time DESC) = 1 ) -SELECT jb.workspace_id, jb.job_id, COALESCE(inv.name, 'job ' || jb.job_id) AS job_name, - inv.trigger_type AS trigger_type, - inv.job_id IS NOT NULL AS listed_in_job_inventory, - runs.run_trigger AS run_trigger_type, - COALESCE(runs.run_count, 0) AS run_count, - jb.job_total_cost, jb.job_all_purpose_cost, jb.job_all_purpose_dbus, - CASE WHEN mc.cluster_id IS NULL OR COALESCE(ct.cost, 0) <= 0 THEN 1.0 ELSE mc.cost / ct.cost END AS job_share_of_cluster, +SELECT t.workspace_id, t.job_id, COALESCE(NULLIF(inv.name, ''), 'job ' || t.job_id) AS job_name, + t.task_key, t.trigger_type, t.cluster_id AS host_cluster_id, + COALESCE(NULLIF(TRIM(c.cluster_name), ''), t.cluster_id) AS host_cluster_name, + t.task_runs, t.task_hours, + COALESCE(cb.cost, 0) AS host_cluster_cost, + COALESCE(cb.dbus, 0) AS host_cluster_dbus, + COALESCE(cb.billed_hours, 0) AS host_cluster_billed_hours, jr.rate AS jobs_rate_per_dbu -FROM job_bill jb -LEFT JOIN runs ON runs.workspace_id = jb.workspace_id AND runs.job_id = jb.job_id -LEFT JOIN inv ON inv.workspace_id = jb.workspace_id AND inv.job_id = jb.job_id -LEFT JOIN main_cluster mc ON mc.workspace_id = jb.workspace_id AND mc.job_id = jb.job_id -LEFT JOIN cluster_total ct ON ct.workspace_id = mc.workspace_id AND ct.cluster_id = mc.cluster_id -LEFT JOIN jobs_rate jr ON jr.workspace_id = jb.workspace_id -WHERE jb.job_all_purpose_cost > 0 -ORDER BY jb.job_all_purpose_cost DESC +FROM task t +JOIN clusters c ON c.workspace_id = t.workspace_id AND c.cluster_id = t.cluster_id +LEFT JOIN inv ON inv.workspace_id = t.workspace_id AND inv.job_id = t.job_id +LEFT JOIN cluster_bill cb ON cb.workspace_id = t.workspace_id AND cb.cluster_id = t.cluster_id +LEFT JOIN jobs_rate jr ON jr.workspace_id = t.workspace_id +WHERE inv.delete_time IS NULL +ORDER BY t.task_hours DESC, t.workspace_id, t.job_id, t.task_key, t.cluster_id, t.trigger_type ` /** Core's rules: a `DatabricksSavingsInput` as JSON in, the report out. */ @@ -362,22 +398,25 @@ export function toClusterFacts(row: Record) { } } -/** One job row as the facts core reads. */ +/** One job task row as the facts core reads. */ export function toJobFacts(row: Record) { const id = String(row.job_id ?? "") + const cluster = String(row.host_cluster_id ?? "") const rate = row.jobs_rate_per_dbu return { job_id: id, workspace_id: row.workspace_id == null ? null : String(row.workspace_id), job_name: String(row.job_name ?? `job ${id}`), + // Core refuses an empty key, and with it every cluster and job fact of the run. + task_key: String(row.task_key || "(no task key)"), trigger_type: row.trigger_type == null ? null : String(row.trigger_type), - listed_in_job_inventory: [true, 1, "true"].includes(row.listed_in_job_inventory as never), - run_trigger_type: row.run_trigger_type == null ? null : String(row.run_trigger_type), - run_count: Math.round(num(row.run_count)), - job_total_cost: num(row.job_total_cost), - job_all_purpose_cost: num(row.job_all_purpose_cost), - job_all_purpose_dbus: num(row.job_all_purpose_dbus), - job_share_of_cluster: Math.min(1, num(row.job_share_of_cluster)), + host_cluster_id: cluster, + host_cluster_name: String(row.host_cluster_name || cluster), + task_runs: Math.round(num(row.task_runs)), + task_hours: num(row.task_hours), + host_cluster_cost: num(row.host_cluster_cost), + host_cluster_dbus: num(row.host_cluster_dbus), + host_cluster_billed_hours: num(row.host_cluster_billed_hours), jobs_rate_per_dbu: rate == null ? null : num(rate), } } @@ -461,13 +500,13 @@ export async function databricksSavings( const binds = [window.end] // A query that fills the row limit reads only its first rows; the result names it. const limited: string[] = [] - const read = async (what: string, sql: string) => { + const read = async (what: string, sql: string, rows = what) => { signal?.throwIfAborted() const left = stopsAt - Date.now() if (left <= 0) throw new Error(`The ${what} query was not started: it would run past ${limitName}.`) try { const result = await connector.execute(sql, ROW_LIMIT, binds, { cancelAfterMs: left, signal }) - if (result.truncated) limited.push(what) + if (result.truncated) limited.push(rows) return records(result) } catch (e) { if (!isQueryTimeoutError(e)) throw e @@ -483,9 +522,9 @@ export async function databricksSavings( } // A table the connection may not read stops only its own check; the rest still report. const skipped: { check: string; reason: string }[] = [] - const optional = async (check: string, what: string, sql: string) => { + const optional = async (check: string, what: string, sql: string, rows = what) => { try { - return await read(what, sql) + return await read(what, sql, rows) } catch (e) { if (signal?.aborted) throw e skipped.push({ check, reason: (e instanceof Error ? e.message : String(e)).split("\n")[0]!.slice(0, 300) }) @@ -493,7 +532,7 @@ export async function databricksSavings( } } const clusters = await optional("Idle Cluster Detection", "all-purpose clusters", DATABRICKS_CLUSTER_FACTS_SQL) - const jobs = await optional("Job Running on All-Purpose Compute", "jobs", DATABRICKS_JOB_FACTS_SQL) + const jobs = await optional("Job Running on All-Purpose Compute", "jobs", DATABRICKS_JOB_FACTS_SQL, "job task rows") const input = { lookback_days: LOOKBACK_DAYS, warehouses: warehouses.map(toFacts), @@ -521,7 +560,8 @@ export async function databricksSavings( ran_on: runsOn, warehouses_checked: warehouses.length, clusters_checked: clusters.length, - jobs_checked: jobs.length, + // The query gives a row per task, cluster and trigger: each job is counted once. + jobs_checked: new Set(jobs.map((row) => JSON.stringify([row.workspace_id ?? null, row.job_id]))).size, skipped, limited, // What each warehouse is and costs now, so a later run can tell what changed. diff --git a/packages/opencode/src/altimate/tools/finops-databricks-savings.ts b/packages/opencode/src/altimate/tools/finops-databricks-savings.ts index ac92c33c77..9cac8b9411 100644 --- a/packages/opencode/src/altimate/tools/finops-databricks-savings.ts +++ b/packages/opencode/src/altimate/tools/finops-databricks-savings.ts @@ -42,6 +42,13 @@ function size(value: unknown): string { const percentOf = (value: unknown) => `${Math.round(Number(value))}%` +/** One decimal; a figure that would round to 0.0 reads "under 0.1". */ +const tenths = (value: unknown) => + Number(value) < 0.05 ? "under 0.1" : Number(value).toLocaleString("en-US", { minimumFractionDigits: 1, maximumFractionDigits: 1 }) + +/** Free text from the workspace on one line, with control and invisible characters gone. */ +const oneLine = (value: unknown) => String(value ?? "").replace(/[\s\p{Cc}\p{Cf}]+/gu, " ").trim() + /** What a finding rests on, one short line each. */ export function evidenceLines(finding: DatabricksSavingsFinding): string[] { const e = finding.evidence as Record & { kind: string } @@ -60,11 +67,11 @@ export function evidenceLines(finding: DatabricksSavingsFinding): string[] { case "idle_cluster": return [ `All-purpose cluster ${percent(e.idle_ratio)} idle across ${e.active_days} active days (${String(e.idle_pattern).replace(/_/g, " ")}), average worker CPU ${percentOf(e.avg_cpu_active_pct)}, peak ${percentOf(e.peak_cpu_active_pct)}`, - `Auto-termination ${e.auto_termination_minutes ? `${e.auto_termination_minutes} minutes` : "off"}; $${Number(e.total_cost).toFixed(2)} billed outside job runs over the window`, + `Auto-termination ${e.auto_termination_minutes ? `${e.auto_termination_minutes} minutes` : "off"}; $${Number(e.total_cost).toFixed(2)} billed on all-purpose compute over the window`, ] case "job_on_all_purpose": return [ - `${e.run_count} runs (${String(e.trigger_type ?? "manual").toLowerCase()}), ${percent(e.all_purpose_share)} of its cost on all-purpose compute, ${percent(e.job_share_of_cluster)} of its cluster's`, + `${Number(e.task_runs).toLocaleString("en-US")} task ${Number(e.task_runs) === 1 ? "run" : "runs"} (${String(e.trigger_type).toLowerCase()}), ${tenths(e.task_hours)} hours on all-purpose compute, mostly cluster ${JSON.stringify(oneLine(e.host_cluster_name))}, about ${tenths(e.estimated_dbus)} DBUs`, `$${Number(e.all_purpose_rate_per_dbu).toFixed(3)} a DBU now; a job cluster would be about $${Number(e.jobs_rate_per_dbu).toFixed(3)}${e.jobs_rate_estimated ? " (estimated: the workspace billed no jobs compute)" : ""}`, ] default: @@ -109,7 +116,7 @@ export function formatSavings(result: DatabricksSavingsResult): string { export const FinopsDatabricksSavingsTool = Tool.define("finops_databricks_savings", { description: - "Check a Databricks workspace for savings with Altimate's rules: SQL warehouses that would cost less as serverless or smaller, idle all-purpose clusters, and scheduled jobs running on all-purpose compute. Read-only queries over seven days of system tables (billing, compute, query history, jobs), covering every workspace those tables show, on the connection's SQL warehouse, which Databricks starts if it is stopped. Returns each finding with its monthly dollar range and evidence, and what each flagged warehouse would cost at other sizes. Ask the user before running it, and pass confirm: true once they agree; without it nothing runs.", + "Check a Databricks workspace for savings with Altimate's rules: SQL warehouses that would cost less as serverless or smaller, idle all-purpose clusters, and scheduled or continuous jobs running on all-purpose clusters. Read-only queries over seven days of system tables (billing, compute, query history, jobs), covering every workspace those tables show, on the connection's SQL warehouse, which Databricks starts if it is stopped. Returns each finding with its monthly dollar range and evidence, and what each flagged warehouse would cost at other sizes. Ask the user before running it, and pass confirm: true once they agree; without it nothing runs.", parameters: z.object({ warehouse: z .string() diff --git a/packages/opencode/test/altimate/finops-databricks-savings.test.ts b/packages/opencode/test/altimate/finops-databricks-savings.test.ts index 211efcec48..6307e299d7 100644 --- a/packages/opencode/test/altimate/finops-databricks-savings.test.ts +++ b/packages/opencode/test/altimate/finops-databricks-savings.test.ts @@ -284,8 +284,21 @@ describe("cluster and job findings", () => { monthly_max: 1530, evidence: { kind: "idle_cluster", idle_pattern: "zombie", idle_ratio: 0.85, auto_termination_minutes: null, active_days: 5, avg_cpu_active_pct: 8, peak_cpu_active_pct: 30, total_cost: 700, wasted_cost: 595 }, }, + { + check_id: "jobs_on_all_purpose_compute", + title: "Job Running on All-Purpose Compute", + resource_type: "job", + resource_id: "42", + resource_name: "nightly", + severity: "medium", + daily_min: 24, + daily_max: 36, + monthly_min: 720, + monthly_max: 1080, + evidence: { kind: "job_on_all_purpose", host_cluster_id: "0101-abc", host_cluster_name: "shared\nanalytics", trigger_type: "CRON", task_runs: 1234, task_hours: 70, estimated_dbus: 1700, all_purpose_rate_per_dbu: 0.55, jobs_rate_per_dbu: 0.33, jobs_rate_estimated: true, wasted_cost: 280 }, + }, ], - total: { monthly_min: 765, monthly_max: 1530, resources: 1 }, + total: { monthly_min: 1485, monthly_max: 2610, resources: 2 }, what_if: [], }, })) @@ -295,7 +308,10 @@ describe("cluster and job findings", () => { expect(result.output).toContain("Not checked: Job Running on All-Purpose Compute ([INSUFFICIENT_PERMISSIONS] system.lakeflow.jobs)") expect(result.output).toContain("shared-analytics (cluster) · Idle Cluster Detection · $765–$1,530 a month (critical)") expect(result.output).toContain("All-purpose cluster 85% idle across 5 active days (zombie), average worker CPU 8%, peak 30%") - expect(result.output).toContain("Auto-termination off; $700.00 billed outside job runs over the window") + expect(result.output).toContain("Auto-termination off; $700.00 billed on all-purpose compute over the window") + expect(result.output).toContain("nightly (job) · Job Running on All-Purpose Compute · $720–$1,080 a month (medium)") + expect(result.output).toContain('1,234 task runs (cron), 70.0 hours on all-purpose compute, mostly cluster "shared analytics", about 1,700.0 DBUs') + expect(result.output).toContain("$0.550 a DBU now; a job cluster would be about $0.330 (estimated: the workspace billed no jobs compute)") }) test("the cluster and job queries each take only the window end, and key every resource by its workspace", () => { @@ -310,33 +326,57 @@ describe("cluster and job findings", () => { expect(DATABRICKS_CLUSTER_FACTS_SQL).toContain("PARTITION BY workspace_id, cluster_id") // Job ids repeat across workspaces: every job key pairs the workspace with the id. expect(DATABRICKS_JOB_FACTS_SQL).toContain("PARTITION BY workspace_id, job_id") - expect(DATABRICKS_JOB_FACTS_SQL).toContain("GROUP BY workspace_id, job_id") - expect(DATABRICKS_JOB_FACTS_SQL).toContain("LEFT JOIN inv ON inv.workspace_id = jb.workspace_id AND inv.job_id = jb.job_id") - expect(DATABRICKS_JOB_FACTS_SQL).toContain("LEFT JOIN jobs_rate jr ON jr.workspace_id = jb.workspace_id") + expect(DATABRICKS_JOB_FACTS_SQL).toContain("GROUP BY tl.workspace_id, tl.job_id, tl.task_key, tl.job_run_id, tl.run_id, tl.period_start_time") + expect(DATABRICKS_JOB_FACTS_SQL).toContain("GROUP BY workspace_id, job_id, task_key, job_run_id, cluster_id") + expect(DATABRICKS_JOB_FACTS_SQL).toContain("GROUP BY workspace_id, job_id, task_key, job_run_id\n") + expect(DATABRICKS_JOB_FACTS_SQL).toContain("GROUP BY tr.workspace_id, tr.job_id, tr.task_key, tr.cluster_id, rt.trigger_type") + expect(DATABRICKS_JOB_FACTS_SQL).toContain("LEFT JOIN inv ON inv.workspace_id = t.workspace_id AND inv.job_id = t.job_id") + expect(DATABRICKS_JOB_FACTS_SQL).toContain("rt.workspace_id = tr.workspace_id AND rt.run_id = tr.job_run_id") + expect(DATABRICKS_JOB_FACTS_SQL).toContain("LEFT JOIN cluster_bill cb ON cb.workspace_id = t.workspace_id AND cb.cluster_id = t.cluster_id") + expect(DATABRICKS_JOB_FACTS_SQL).toContain("LEFT JOIN jobs_rate jr ON jr.workspace_id = t.workspace_id") + }) + + test("the job query finds runs through the compute they used, on live all-purpose clusters, for jobs not deleted", () => { + // Billing leaves the job id empty on all-purpose compute, so it cannot find these jobs. + expect(DATABRICKS_JOB_FACTS_SQL).not.toContain("usage_metadata.job_id") + expect(DATABRICKS_JOB_FACTS_SQL).toContain("explode(t.compute_ids) AS compute_id") + expect(DATABRICKS_JOB_FACTS_SQL).toContain("c.workspace_id = tl.workspace_id AND c.cluster_id = tl.compute_id") + expect(DATABRICKS_JOB_FACTS_SQL).toContain("WHERE delete_time IS NULL AND cluster_source IN ('UI', 'API')") + expect(DATABRICKS_JOB_FACTS_SQL).toContain("WHERE inv.delete_time IS NULL") + // A row naming two all-purpose clusters counts once, and a run's hours add up its hourly rows. + expect(DATABRICKS_JOB_FACTS_SQL).toContain("MAX(timestampdiff(SECOND, tl.period_start_time, tl.period_end_time)) AS seconds") + expect(DATABRICKS_JOB_FACTS_SQL).toContain("SUM(seconds) / 3600.0 AS hours") + // A run is credited to the cluster it spent most of its time on, the lowest id on a tie. + expect(DATABRICKS_JOB_FACTS_SQL).toContain("MIN_BY(cluster_id, struct(-seconds, cluster_id)) AS cluster_id") + // Latest inventory rows, the window on both timelines, one duration per billed period, the + // jobs rate from jobs compute, and the largest rows first when the row limit cuts the list. + expect(DATABRICKS_JOB_FACTS_SQL.match(/ORDER BY change_time DESC\) = 1/g)).toHaveLength(2) + expect(DATABRICKS_JOB_FACTS_SQL.match(/period_start_time >= CAST\(DATE_SUB\(p\.we_date, 7\) AS TIMESTAMP\) AND \w+\.period_start_time < CAST\(p\.we_date AS TIMESTAMP\)/g)).toHaveLength(2) + expect(DATABRICKS_JOB_FACTS_SQL).toContain("GROUP BY workspace_id, cluster_id, usage_start_time, usage_end_time") + expect(DATABRICKS_JOB_FACTS_SQL).toContain("FROM bill WHERE product = 'JOBS' GROUP BY workspace_id") + expect(DATABRICKS_JOB_FACTS_SQL).toContain("ORDER BY t.task_hours DESC") + // A task run with no parent run id is its own run, so it still finds its trigger. + expect(DATABRICKS_JOB_FACTS_SQL).toContain("COALESCE(NULLIF(t.job_run_id, ''), t.run_id) AS job_run_id") + // The thresholds stay in core: every trigger is measured and handed over. + expect(DATABRICKS_JOB_FACTS_SQL).not.toMatch(/CRON|PERIODIC|CONTINUOUS|HAVING/) }) test("cluster and job rows are coerced to the facts core reads", () => { const cluster = toClusterFacts({ cluster_id: "c1", cluster_name: null, auto_termination_minutes: "120", total_node_minutes: 900n, active_util_days: "4", total_worker_minutes: 900, idle_node_minutes: 450, total_cost: "70.5" }) expect(cluster).toMatchObject({ cluster_id: "c1", cluster_name: "c1", auto_termination_minutes: 120, total_node_minutes: 900, active_util_days: 4, total_cost: 70.5 }) expect(toClusterFacts({ cluster_id: "c2" }).auto_termination_minutes).toBeNull() - const job = toJobFacts({ job_id: "j1", job_name: "nightly", trigger_type: null, listed_in_job_inventory: true, run_trigger_type: "ONE_TIME", run_count: 7n, job_share_of_cluster: 1.2, jobs_rate_per_dbu: null }) - expect(job).toMatchObject({ job_id: "j1", trigger_type: null, listed_in_job_inventory: true, run_trigger_type: "ONE_TIME", run_count: 7, job_share_of_cluster: 1, jobs_rate_per_dbu: null }) - // A job the inventory does not list arrives with false, or with nothing at all. - expect(toJobFacts({ job_id: "j2", listed_in_job_inventory: false }).listed_in_job_inventory).toBe(false) - expect(toJobFacts({ job_id: "j3" })).toMatchObject({ listed_in_job_inventory: false, run_trigger_type: null }) - }) - - test("the job query hands the inventory's trigger and the runs' trigger to core separately", () => { - expect(DATABRICKS_JOB_FACTS_SQL).toContain("inv.trigger_type AS trigger_type") - expect(DATABRICKS_JOB_FACTS_SQL).toContain("inv.job_id IS NOT NULL AS listed_in_job_inventory") - expect(DATABRICKS_JOB_FACTS_SQL).toContain("runs.run_trigger AS run_trigger_type") - expect(DATABRICKS_JOB_FACTS_SQL).not.toContain("COALESCE(inv.trigger_type") + const job = toJobFacts({ workspace_id: 222n, job_id: 42n, job_name: "nightly", task_key: "main", trigger_type: "CRON", host_cluster_id: "c1", host_cluster_name: null, task_runs: 7n, task_hours: "3.5", host_cluster_cost: "55", host_cluster_dbus: 100, host_cluster_billed_hours: "10", jobs_rate_per_dbu: null }) + expect(job).toEqual({ job_id: "42", workspace_id: "222", job_name: "nightly", task_key: "main", trigger_type: "CRON", host_cluster_id: "c1", host_cluster_name: "c1", task_runs: 7, task_hours: 3.5, host_cluster_cost: 55, host_cluster_dbus: 100, host_cluster_billed_hours: 10, jobs_rate_per_dbu: null }) + expect(toJobFacts({ job_id: "j", host_cluster_id: "c1", host_cluster_name: "" }).host_cluster_name).toBe("c1") + expect(toJobFacts({ job_id: "j", task_key: "" }).task_key).toBe("(no task key)") + // A run whose trigger the timeline does not give stays unknown, and core does not count it. + expect(toJobFacts({ job_id: "j2", task_key: "t", host_cluster_id: "c" })).toMatchObject({ trigger_type: null, job_name: "job j2", task_runs: 0, host_cluster_dbus: 0 }) }) }) describe("databricks savings handler over a stubbed connection", () => { const WAREHOUSE_ROW = { workspace_id: "111", warehouse_id: "abc123", warehouse_name: "BI", warehouse_type: "PRO", warehouse_size: "SMALL", total_cost: 27.5 } - const JOB_ROW = { workspace_id: "222", job_id: "j1", job_name: "nightly", trigger_type: null, listed_in_job_inventory: true, run_trigger_type: "ONE_TIME", run_count: 9 } + const JOB_ROW = { workspace_id: "222", job_id: "j1", job_name: "nightly", task_key: "main", trigger_type: "CRON", host_cluster_id: "c1", host_cluster_name: "shared", task_runs: 9, task_hours: 4.5, host_cluster_cost: 55, host_cluster_dbus: 100, host_cluster_billed_hours: 10, jobs_rate_per_dbu: 0.15 } const table = (row: Record, truncated = false) => ({ columns: Object.keys(row), rows: [Object.values(row)], truncated }) let calls: { sql: string; limit: number; binds: unknown[]; timeout?: number; clientTimeout?: number; signal?: AbortSignal }[] let sent: any @@ -390,10 +430,25 @@ describe("databricks savings handler over a stubbed connection", () => { expect(sent.warehouses[0]).toMatchObject({ workspace_id: "111", warehouse_id: "abc123", warehouse_size: "SMALL", total_cost: 27.5 }) expect(result.warehouses?.[0]).toMatchObject({ id: "abc123", workspace_id: "111" }) expect(sent.clusters).toEqual([]) - expect(sent.jobs[0]).toMatchObject({ workspace_id: "222", job_id: "j1", trigger_type: null, listed_in_job_inventory: true, run_trigger_type: "ONE_TIME" }) + expect(sent.jobs[0]).toMatchObject({ workspace_id: "222", job_id: "j1", task_key: "main", trigger_type: "CRON", host_cluster_id: "c1", task_runs: 9, task_hours: 4.5 }) expect(result.limited).toEqual([]) }) + test("a job counts once however many task rows it has, and the same id in two workspaces is two jobs", async () => { + const row = (workspace: string, task: string) => Object.values({ ...JOB_ROW, workspace_id: workspace, task_key: task }) + const spy = connect((sql) => + sql === DATABRICKS_WAREHOUSE_FACTS_SQL + ? table(WAREHOUSE_ROW) + : sql === DATABRICKS_JOB_FACTS_SQL + ? { columns: Object.keys(JOB_ROW), rows: [row("222", "a"), row("222", "b"), row("333", "a")], truncated: false } + : { columns: [], rows: [], truncated: false }, + ) + const result = await databricksSavings({ warehouse: "dbx", now_ms: Date.parse("2026-11-18T10:00:00Z") }, rules) + spy.mockRestore() + expect(sent.jobs).toHaveLength(3) + expect(result.jobs_checked).toBe(2) + }) + test("a query past its limit fails the check with a plain reason, or skips an optional check", async () => { const timeout = (cancelled: boolean) => Object.assign(new Error("QUERY_TIMEOUT: databricks query exceeded"), { code: "QUERY_TIMEOUT", cancelled }) let spy = connect((sql) => { @@ -482,8 +537,8 @@ describe("databricks savings handler over a stubbed connection", () => { const result = await databricksSavings({ warehouse: "dbx", now_ms: Date.parse("2026-11-18T10:00:00Z") }, rules) spy.mockRestore() expect(calls.every((one) => one.limit === 10_000)).toBe(true) - expect(result.limited).toEqual(["jobs"]) - expect(formatSavings(result)).toContain("Only the first 10,000 jobs were read; any beyond them were not checked.") + expect(result.limited).toEqual(["job task rows"]) + expect(formatSavings(result)).toContain("Only the first 10,000 job task rows were read; any beyond them were not checked.") }) test("when core refuses the cluster or job facts, the warehouse findings still report and those checks are named", async () => { @@ -492,7 +547,7 @@ describe("databricks savings handler over a stubbed connection", () => { const picky = (input: string) => { const parsed = JSON.parse(input) calls.push(parsed) - if (parsed.clusters.length > 0 || parsed.jobs.length > 0) throw new Error("invalid input: jobs[0].job_total_cost must not be negative") + if (parsed.clusters.length > 0 || parsed.jobs.length > 0) throw new Error("invalid input: jobs[0].task_hours must not be negative") return rules(input) } const result = await databricksSavings({ warehouse: "dbx", now_ms: Date.parse("2026-11-18T10:00:00Z") }, picky) @@ -501,7 +556,7 @@ describe("databricks savings handler over a stubbed connection", () => { expect(calls.length).toBe(2) expect(sent.warehouses[0].warehouse_id).toBe("abc123") expect(result.skipped?.map((one) => one.check)).toEqual(["Idle Cluster Detection", "Job Running on All-Purpose Compute"]) - expect(result.skipped?.[0]?.reason).toContain("core refused their facts: invalid input: jobs[0].job_total_cost") + expect(result.skipped?.[0]?.reason).toContain("core refused their facts: invalid input: jobs[0].task_hours") expect([result.clusters_checked, result.jobs_checked]).toEqual([0, 0]) }) From b468212fcef0d1051866284bf8defc997246b2cf Mon Sep 17 00:00:00 2001 From: Shreyas Telkar Date: Wed, 18 Nov 2026 15:04:53 -0800 Subject: [PATCH 22/34] fix: job facts skip rows with no job, keep every task, and count job ids of any type A timeline row with no job is left out and a task with no key gets one, since core refuses empty ids. The idle check drops a billing filter that never matched on all-purpose compute. Job and cluster names stay on one line, and the evidence says the cluster took most of the DBUs. Co-Authored-By: Claude Opus 5.5 --- .../native/finops/databricks-savings.ts | 13 ++-- .../opencode/src/altimate/native/types.ts | 2 +- .../tools/finops-databricks-savings.ts | 12 ++-- .../finops-databricks-savings.test.ts | 61 ++++++++++++++++++- 4 files changed, 74 insertions(+), 14 deletions(-) diff --git a/packages/opencode/src/altimate/native/finops/databricks-savings.ts b/packages/opencode/src/altimate/native/finops/databricks-savings.ts index 1fdf386e60..d0c2a012ff 100644 --- a/packages/opencode/src/altimate/native/finops/databricks-savings.ts +++ b/packages/opencode/src/altimate/native/finops/databricks-savings.ts @@ -203,7 +203,7 @@ bill AS ( FROM system.billing.usage u CROSS JOIN p ${LIST_PRICE_JOIN} WHERE u.billing_origin_product = 'ALL_PURPOSE' - AND u.usage_metadata.cluster_id IS NOT NULL AND u.usage_metadata.job_id IS NULL + AND u.usage_metadata.cluster_id IS NOT NULL AND u.usage_date >= DATE_SUB(p.we_date, 7) AND u.usage_date < p.we_date GROUP BY u.workspace_id, u.usage_metadata.cluster_id ) @@ -241,10 +241,12 @@ clusters AS ( WHERE delete_time IS NULL AND cluster_source IN ('UI', 'API') ), timeline AS ( - SELECT t.workspace_id, t.job_id, t.task_key, COALESCE(NULLIF(t.job_run_id, ''), t.run_id) AS job_run_id, + SELECT t.workspace_id, t.job_id, COALESCE(NULLIF(TRIM(t.task_key), ''), '(no task key)') AS task_key, + COALESCE(NULLIF(t.job_run_id, ''), t.run_id) AS job_run_id, t.run_id, t.period_start_time, t.period_end_time, explode(t.compute_ids) AS compute_id FROM system.lakeflow.job_task_run_timeline t CROSS JOIN p WHERE t.period_start_time >= CAST(DATE_SUB(p.we_date, 7) AS TIMESTAMP) AND t.period_start_time < CAST(p.we_date AS TIMESTAMP) + AND t.job_id IS NOT NULL AND t.job_id <> '' ), timeline_row AS ( SELECT tl.workspace_id, tl.job_id, tl.task_key, tl.job_run_id, tl.run_id, tl.period_start_time, @@ -306,7 +308,7 @@ inv AS ( SELECT workspace_id, job_id, name, delete_time FROM system.lakeflow.jobs QUALIFY ROW_NUMBER() OVER (PARTITION BY workspace_id, job_id ORDER BY change_time DESC) = 1 ) -SELECT t.workspace_id, t.job_id, COALESCE(NULLIF(inv.name, ''), 'job ' || t.job_id) AS job_name, +SELECT t.workspace_id, t.job_id, COALESCE(NULLIF(TRIM(inv.name), ''), 'job ' || t.job_id) AS job_name, t.task_key, t.trigger_type, t.cluster_id AS host_cluster_id, COALESCE(NULLIF(TRIM(c.cluster_name), ''), t.cluster_id) AS host_cluster_name, t.task_runs, t.task_hours, @@ -407,8 +409,7 @@ export function toJobFacts(row: Record) { job_id: id, workspace_id: row.workspace_id == null ? null : String(row.workspace_id), job_name: String(row.job_name ?? `job ${id}`), - // Core refuses an empty key, and with it every cluster and job fact of the run. - task_key: String(row.task_key || "(no task key)"), + task_key: String(row.task_key ?? ""), trigger_type: row.trigger_type == null ? null : String(row.trigger_type), host_cluster_id: cluster, host_cluster_name: String(row.host_cluster_name || cluster), @@ -561,7 +562,7 @@ export async function databricksSavings( warehouses_checked: warehouses.length, clusters_checked: clusters.length, // The query gives a row per task, cluster and trigger: each job is counted once. - jobs_checked: new Set(jobs.map((row) => JSON.stringify([row.workspace_id ?? null, row.job_id]))).size, + jobs_checked: new Set(jobs.map((row) => `${row.workspace_id ?? ""}/${row.job_id}`)).size, skipped, limited, // What each warehouse is and costs now, so a later run can tell what changed. diff --git a/packages/opencode/src/altimate/native/types.ts b/packages/opencode/src/altimate/native/types.ts index 201f402b7b..c221a4cba9 100644 --- a/packages/opencode/src/altimate/native/types.ts +++ b/packages/opencode/src/altimate/native/types.ts @@ -691,7 +691,7 @@ export interface DatabricksSavingsResult { jobs_checked?: number /** Checks whose system tables the connection could not read. */ skipped?: { check: string; reason: string }[] - /** What a query read only the first 10,000 of, e.g. `jobs`. */ + /** What a query read only the first 10,000 of, e.g. `job task rows`. */ limited?: string[] /** Each SQL warehouse as it is now, and its cost over the window. */ warehouses?: { id: string; workspace_id?: string | null; name: string; type: string; size: string | null; window_cost: number }[] diff --git a/packages/opencode/src/altimate/tools/finops-databricks-savings.ts b/packages/opencode/src/altimate/tools/finops-databricks-savings.ts index 9cac8b9411..4b8cdf539d 100644 --- a/packages/opencode/src/altimate/tools/finops-databricks-savings.ts +++ b/packages/opencode/src/altimate/tools/finops-databricks-savings.ts @@ -47,7 +47,11 @@ const tenths = (value: unknown) => Number(value) < 0.05 ? "under 0.1" : Number(value).toLocaleString("en-US", { minimumFractionDigits: 1, maximumFractionDigits: 1 }) /** Free text from the workspace on one line, with control and invisible characters gone. */ -const oneLine = (value: unknown) => String(value ?? "").replace(/[\s\p{Cc}\p{Cf}]+/gu, " ").trim() +const oneLine = (value: unknown) => + String(value ?? "") + .replace(/\p{Cf}/gu, "") + .replace(/[\s\p{Cc}]+/gu, " ") + .trim() /** What a finding rests on, one short line each. */ export function evidenceLines(finding: DatabricksSavingsFinding): string[] { @@ -71,7 +75,7 @@ export function evidenceLines(finding: DatabricksSavingsFinding): string[] { ] case "job_on_all_purpose": return [ - `${Number(e.task_runs).toLocaleString("en-US")} task ${Number(e.task_runs) === 1 ? "run" : "runs"} (${String(e.trigger_type).toLowerCase()}), ${tenths(e.task_hours)} hours on all-purpose compute, mostly cluster ${JSON.stringify(oneLine(e.host_cluster_name))}, about ${tenths(e.estimated_dbus)} DBUs`, + `${Number(e.task_runs).toLocaleString("en-US")} task ${Number(e.task_runs) === 1 ? "run" : "runs"} (${String(e.trigger_type).toLowerCase()}), ${tenths(e.task_hours)} hours on all-purpose compute, ${Number(e.estimated_dbus) < 0.05 ? "under 0.1" : `about ${tenths(e.estimated_dbus)}`} DBUs, most on cluster ${JSON.stringify(oneLine(e.host_cluster_name) || oneLine(e.host_cluster_id))}`, `$${Number(e.all_purpose_rate_per_dbu).toFixed(3)} a DBU now; a job cluster would be about $${Number(e.jobs_rate_per_dbu).toFixed(3)}${e.jobs_rate_estimated ? " (estimated: the workspace billed no jobs compute)" : ""}`, ] default: @@ -100,12 +104,12 @@ export function formatSavings(result: DatabricksSavingsResult): string { ) for (const finding of report.findings) { lines.push( - `${finding.resource_name} (${finding.resource_type.replace("_", " ")}${finding.workspace_id ? `, workspace ${finding.workspace_id}` : ""}) · ${finding.title} · ${range(finding.monthly_min, finding.monthly_max)} a month (${finding.severity})`, + `${oneLine(finding.resource_name) || finding.resource_id} (${finding.resource_type.replace("_", " ")}${finding.workspace_id ? `, workspace ${finding.workspace_id}` : ""}) · ${finding.title} · ${range(finding.monthly_min, finding.monthly_max)} a month (${finding.severity})`, ) for (const line of evidenceLines(finding)) lines.push(` ${line}`) } for (const option of report.what_if) { - lines.push("", `What ${option.resource_name} (now ${option.current_label}, ${dollars(option.current_monthly_cost)} a month) would cost instead:`) + lines.push("", `What ${oneLine(option.resource_name) || option.resource_id} (now ${option.current_label}, ${dollars(option.current_monthly_cost)} a month) would cost instead:`) for (const choice of option.choices) { const fit = choice.fits ? "" : ` (too small: ${choice.cores} cores, its queries need ${choice.cores_needed.toFixed(0)})` lines.push(` ${size(choice.label)}: ${dollars(choice.monthly_cost)} a month, saving ${dollars(choice.monthly_saving)}${fit}`) diff --git a/packages/opencode/test/altimate/finops-databricks-savings.test.ts b/packages/opencode/test/altimate/finops-databricks-savings.test.ts index 6307e299d7..18b1cffcb8 100644 --- a/packages/opencode/test/altimate/finops-databricks-savings.test.ts +++ b/packages/opencode/test/altimate/finops-databricks-savings.test.ts @@ -310,10 +310,37 @@ describe("cluster and job findings", () => { expect(result.output).toContain("All-purpose cluster 85% idle across 5 active days (zombie), average worker CPU 8%, peak 30%") expect(result.output).toContain("Auto-termination off; $700.00 billed on all-purpose compute over the window") expect(result.output).toContain("nightly (job) · Job Running on All-Purpose Compute · $720–$1,080 a month (medium)") - expect(result.output).toContain('1,234 task runs (cron), 70.0 hours on all-purpose compute, mostly cluster "shared analytics", about 1,700.0 DBUs') + expect(result.output).toContain('1,234 task runs (cron), 70.0 hours on all-purpose compute, about 1,700.0 DBUs, most on cluster "shared analytics"') expect(result.output).toContain("$0.550 a DBU now; a job cluster would be about $0.330 (estimated: the workspace billed no jobs compute)") }) + test("a job's evidence reads right for one short run, and workspace names stay on one line", async () => { + const finding = { + check_id: "jobs_on_all_purpose_compute", title: "Job Running on All-Purpose Compute", resource_type: "job", resource_id: "42", resource_name: "nightly\nTotal: $0", severity: "medium", + daily_min: 1, daily_max: 2, monthly_min: 30, monthly_max: 60, + evidence: { kind: "job_on_all_purpose", host_cluster_id: "c", host_cluster_name: " a\n\u001b[2J\u200bb ", trigger_type: "PERIODIC", task_runs: 1, task_hours: 0.01, estimated_dbus: 0.02, all_purpose_rate_per_dbu: 0.55, jobs_rate_per_dbu: 0.15, jobs_rate_estimated: false, wasted_cost: 7 }, + } + Dispatcher.register("finops.databricks_savings" as any, async () => ({ ...REPORT, report: { lookback_days: 7, findings: [finding], total: { monthly_min: 30, monthly_max: 60, resources: 1 }, what_if: [] } })) + const tool = await initTool(FinopsDatabricksSavingsTool) + const result = await tool.execute({ confirm: true }, stubCtx()) + expect(result.output).toContain('1 task run (periodic), under 0.1 hours on all-purpose compute, under 0.1 DBUs, most on cluster "a [2Jb"') + expect(result.output).toContain("nightly Total: $0 (job)") + // A name with nothing printable falls back to the id, for the job and for its cluster. + Dispatcher.register("finops.databricks_savings" as any, async () => ({ + ...REPORT, + report: { + lookback_days: 7, + findings: [{ ...finding, resource_name: " \u200b", evidence: { ...finding.evidence, host_cluster_name: "\u200b" } }], + total: { monthly_min: 30, monthly_max: 60, resources: 1 }, + what_if: [{ resource_id: "wh1", resource_name: "BI\nTotal: $0", current_label: "SMALL (PRO)", current_monthly_cost: 10, choices: [] }], + }, + })) + const unnamed = await (await initTool(FinopsDatabricksSavingsTool)).execute({ confirm: true }, stubCtx()) + expect(unnamed.output).toContain("42 (job)") + expect(unnamed.output).toContain('most on cluster "c"') + expect(unnamed.output).toContain("What BI Total: $0 (now") + }) + test("the cluster and job queries each take only the window end, and key every resource by its workspace", () => { for (const sql of [DATABRICKS_CLUSTER_FACTS_SQL, DATABRICKS_JOB_FACTS_SQL]) { expect(sql.trim()).toMatch(/^WITH\b/) @@ -339,6 +366,7 @@ describe("cluster and job findings", () => { test("the job query finds runs through the compute they used, on live all-purpose clusters, for jobs not deleted", () => { // Billing leaves the job id empty on all-purpose compute, so it cannot find these jobs. expect(DATABRICKS_JOB_FACTS_SQL).not.toContain("usage_metadata.job_id") + expect(DATABRICKS_CLUSTER_FACTS_SQL).not.toContain("usage_metadata.job_id") expect(DATABRICKS_JOB_FACTS_SQL).toContain("explode(t.compute_ids) AS compute_id") expect(DATABRICKS_JOB_FACTS_SQL).toContain("c.workspace_id = tl.workspace_id AND c.cluster_id = tl.compute_id") expect(DATABRICKS_JOB_FACTS_SQL).toContain("WHERE delete_time IS NULL AND cluster_source IN ('UI', 'API')") @@ -355,7 +383,19 @@ describe("cluster and job findings", () => { expect(DATABRICKS_JOB_FACTS_SQL).toContain("GROUP BY workspace_id, cluster_id, usage_start_time, usage_end_time") expect(DATABRICKS_JOB_FACTS_SQL).toContain("FROM bill WHERE product = 'JOBS' GROUP BY workspace_id") expect(DATABRICKS_JOB_FACTS_SQL).toContain("ORDER BY t.task_hours DESC") - // A task run with no parent run id is its own run, so it still finds its trigger. + // A timeline row with no job is left out, and a task with no key still gets one: core refuses + // an empty id, and would take every cluster and job fact of the run with it. + expect(DATABRICKS_JOB_FACTS_SQL).toContain("t.job_id IS NOT NULL AND t.job_id <> ''") + expect(DATABRICKS_JOB_FACTS_SQL).toContain("COALESCE(NULLIF(TRIM(t.task_key), ''), '(no task key)') AS task_key") + // The arithmetic each output column rests on, and every column the row mapping reads. + expect(DATABRICKS_JOB_FACTS_SQL).toContain("SUM(seconds) AS seconds") + expect(DATABRICKS_JOB_FACTS_SQL).toContain("COUNT(*) AS task_runs, SUM(tr.hours) AS task_hours") + expect(DATABRICKS_JOB_FACTS_SQL).toContain("SUM(cost) AS cost, SUM(dbus) AS dbus, SUM(seconds) / 3600.0 AS billed_hours") + expect(DATABRICKS_JOB_FACTS_SQL).toContain("FROM bill WHERE product = 'ALL_PURPOSE' AND cluster_id IS NOT NULL") + for (const column of ["job_name", "host_cluster_id", "host_cluster_name", "host_cluster_cost", "host_cluster_dbus", "host_cluster_billed_hours", "jobs_rate_per_dbu"]) { + expect(DATABRICKS_JOB_FACTS_SQL).toContain(`AS ${column}`) + } + // A task run with no parent run id counts as its own run, as Altimate's hosted check reads it. expect(DATABRICKS_JOB_FACTS_SQL).toContain("COALESCE(NULLIF(t.job_run_id, ''), t.run_id) AS job_run_id") // The thresholds stay in core: every trigger is measured and handed over. expect(DATABRICKS_JOB_FACTS_SQL).not.toMatch(/CRON|PERIODIC|CONTINUOUS|HAVING/) @@ -368,7 +408,6 @@ describe("cluster and job findings", () => { const job = toJobFacts({ workspace_id: 222n, job_id: 42n, job_name: "nightly", task_key: "main", trigger_type: "CRON", host_cluster_id: "c1", host_cluster_name: null, task_runs: 7n, task_hours: "3.5", host_cluster_cost: "55", host_cluster_dbus: 100, host_cluster_billed_hours: "10", jobs_rate_per_dbu: null }) expect(job).toEqual({ job_id: "42", workspace_id: "222", job_name: "nightly", task_key: "main", trigger_type: "CRON", host_cluster_id: "c1", host_cluster_name: "c1", task_runs: 7, task_hours: 3.5, host_cluster_cost: 55, host_cluster_dbus: 100, host_cluster_billed_hours: 10, jobs_rate_per_dbu: null }) expect(toJobFacts({ job_id: "j", host_cluster_id: "c1", host_cluster_name: "" }).host_cluster_name).toBe("c1") - expect(toJobFacts({ job_id: "j", task_key: "" }).task_key).toBe("(no task key)") // A run whose trigger the timeline does not give stays unknown, and core does not count it. expect(toJobFacts({ job_id: "j2", task_key: "t", host_cluster_id: "c" })).toMatchObject({ trigger_type: null, job_name: "job j2", task_runs: 0, host_cluster_dbus: 0 }) }) @@ -447,6 +486,22 @@ describe("databricks savings handler over a stubbed connection", () => { spy.mockRestore() expect(sent.jobs).toHaveLength(3) expect(result.jobs_checked).toBe(2) + expect(result.success).toBe(true) + }) + + test("job ids of any type count, so a finished run cannot fail on them", async () => { + const row = (job: unknown) => Object.values({ ...JOB_ROW, workspace_id: 222n, job_id: job }) + const spy = connect((sql) => + sql === DATABRICKS_WAREHOUSE_FACTS_SQL + ? table(WAREHOUSE_ROW) + : sql === DATABRICKS_JOB_FACTS_SQL + ? { columns: Object.keys(JOB_ROW), rows: [row(42n), row(43n)], truncated: false } + : { columns: [], rows: [], truncated: false }, + ) + const result = await databricksSavings({ warehouse: "dbx", now_ms: Date.parse("2026-11-18T10:00:00Z") }, rules) + spy.mockRestore() + expect(result.success).toBe(true) + expect(result.jobs_checked).toBe(2) }) test("a query past its limit fails the check with a plain reason, or skips an optional check", async () => { From 20b89b311a2636ff62e97c2be18dba71f02b73c6 Mon Sep 17 00:00:00 2001 From: Shreyas Telkar Date: Thu, 19 Nov 2026 15:39:14 -0800 Subject: [PATCH 23/34] fix: close a Databricks statement whose fetch fails, and test the missing-rules refusal - `runStatement` closes the operation when `fetchAll()` rejects, with and without a deadline, and reports the fetch error even if the close also fails. - A test runs the savings handler with a core that has no `databricksSavings` export and checks it refuses before opening a warehouse connection. Co-Authored-By: Claude Opus 5.5 --- packages/drivers/src/databricks.ts | 14 ++++++- .../drivers/test/databricks-deadline.test.ts | 41 +++++++++++++++++++ .../finops-databricks-savings.test.ts | 16 ++++++++ 3 files changed, 70 insertions(+), 1 deletion(-) diff --git a/packages/drivers/src/databricks.ts b/packages/drivers/src/databricks.ts index baa47c62a1..306e6f4549 100644 --- a/packages/drivers/src/databricks.ts +++ b/packages/drivers/src/databricks.ts @@ -49,7 +49,15 @@ export async function connect(config: ConnectionConfig): Promise { const deadlineMs = options?.cancelAfterMs if (!deadlineMs || deadlineMs <= 0) { const operation = await session.executeStatement(query, stmtOptions) - const rows = await operation.fetchAll() + let rows: any[] + try { + rows = await operation.fetchAll() + } catch (error) { + // A statement that failed while fetching is still open on the session: free it, and + // report the fetch's own error rather than any from the close. + operation.close().catch(() => {}) + throw error + } await operation.close() return rows } @@ -84,6 +92,10 @@ export async function connect(config: ConnectionConfig): Promise { let outcome: any[] | typeof STOPPED try { outcome = await Promise.race([work, stopping]) + } catch (error) { + // The fetch failed before any stop: the statement is still open on the session. + operation?.close().catch(() => {}) + throw error } finally { clearTimeout(timer) signal?.removeEventListener("abort", onAbort) diff --git a/packages/drivers/test/databricks-deadline.test.ts b/packages/drivers/test/databricks-deadline.test.ts index 7c50f58d82..4f262da714 100644 --- a/packages/drivers/test/databricks-deadline.test.ts +++ b/packages/drivers/test/databricks-deadline.test.ts @@ -12,6 +12,7 @@ await restoreModuleMocksAfterTests("@databricks/sql") let fetch: () => Promise let startDelayMs = 0 let cancelResult: () => Promise +let closeResult: () => Promise let cancels = 0 let closes = 0 @@ -19,6 +20,7 @@ beforeEach(() => { fetch = async () => [{ n: 1 }] startDelayMs = 0 cancelResult = async () => ({}) + closeResult = async () => ({}) cancels = 0 closes = 0 }) @@ -39,6 +41,7 @@ class FakeClient { }, close: async () => { closes++ + return closeResult() }, } }, @@ -142,3 +145,41 @@ describe("Databricks server-side cancel", () => { expect(cancels).toBe(0) }) }) + +describe("Databricks statement cleanup when the fetch fails", () => { + const failingFetch = async (): Promise => { + throw new Error("[TABLE_OR_VIEW_NOT_FOUND] system.x") + } + + for (const [label, options] of [ + ["with a deadline", { cancelAfterMs: 5_000 }], + ["without a deadline", { queryTimeoutMs: 5_000 }], + ] as const) { + test(`${label}, the statement is closed and the fetch's error is reported`, async () => { + fetch = failingFetch + const db = await connector() + const error = await db.execute("SELECT 1", 10, [], options).catch((e) => e) + expect(error.message).toContain("TABLE_OR_VIEW_NOT_FOUND") + expect(isQueryTimeoutError(error)).toBe(false) + expect([cancels, closes]).toEqual([0, 1]) + }) + + test(`${label}, a close that also fails does not replace the fetch's error`, async () => { + fetch = failingFetch + closeResult = async () => { + throw new Error("close failed") + } + const db = await connector() + const error = await db.execute("SELECT 1", 10, [], options).catch((e) => e) + expect(error.message).toContain("TABLE_OR_VIEW_NOT_FOUND") + expect(closes).toBe(1) + }) + } + + test("a statement that returns rows without a deadline is closed once", async () => { + const db = await connector() + const result = await db.execute("SELECT 1", 10, []) + expect(result.rows).toEqual([[1]]) + expect([cancels, closes]).toEqual([0, 1]) + }) +}) diff --git a/packages/opencode/test/altimate/finops-databricks-savings.test.ts b/packages/opencode/test/altimate/finops-databricks-savings.test.ts index 18b1cffcb8..493672f1fe 100644 --- a/packages/opencode/test/altimate/finops-databricks-savings.test.ts +++ b/packages/opencode/test/altimate/finops-databricks-savings.test.ts @@ -257,6 +257,22 @@ describe("databricks savings handler", () => { expect(result.success).toBe(false) expect(result.error).toContain("/sql/1.0/warehouses/") }) + + test("a core without the savings rules is refused before connecting to the warehouse", async () => { + Registry.setConfigs({ + dbx: { type: "databricks", server_hostname: "dbc-0.cloud.databricks.com", http_path: "/sql/1.0/warehouses/abc123", access_token: "x" }, + }) + const get = spyOn(Registry, "get").mockRejectedValue(new Error("opened a connection")) + // A published core that predates the rules exports `initSdk` but not `databricksSavings`. + const result = await databricksSavings({ warehouse: "dbx" }, rulesEngine({ initSdk: () => {} })) + const connections = get.mock.calls.length + get.mockRestore() + expect(result.success).toBe(false) + expect(result.error).toBe( + "This altimate-code build's altimate-core has no Databricks savings rules. Update altimate-code, then try again.", + ) + expect(connections).toBe(0) + }) }) describe("cluster and job findings", () => { From 16d0af70c01ae07d7046a9fca801da8ad25c9863 Mon Sep 17 00:00:00 2001 From: Shreyas Telkar Date: Wed, 18 Nov 2026 14:49:31 -0800 Subject: [PATCH 24/34] feat: the Snowflake driver cancels on `cancelAfterMs` or abort, and stays off cloud metadata in assessment mode - Past the deadline, or when the signal aborts, the statement is cancelled on Snowflake. One still waiting on a reconnect is never sent. Calls without `cancelAfterMs` run as before. - When it loads, snowflake-sdk probes the AWS, Azure and GCP metadata and identity services. A run that promises to contact only the warehouse now turns that off. Co-Authored-By: Claude Opus 5.5 --- packages/drivers/src/snowflake.ts | 107 +++++++- packages/drivers/src/types.ts | 2 +- .../drivers/test/snowflake-deadline.test.ts | 238 ++++++++++++++++++ 3 files changed, 336 insertions(+), 11 deletions(-) create mode 100644 packages/drivers/test/snowflake-deadline.test.ts diff --git a/packages/drivers/src/snowflake.ts b/packages/drivers/src/snowflake.ts index 24aba33688..384ad2c526 100644 --- a/packages/drivers/src/snowflake.ts +++ b/packages/drivers/src/snowflake.ts @@ -5,6 +5,8 @@ import * as fs from "fs" import type { ConnectionConfig, Connector, ConnectorResult, ExecuteOptions, SchemaColumn } from "./types" import { toPkcs8Pem } from "./snowflake-key" +import { QueryTimeoutError } from "./types" +import { MAX_TIMER_MS } from "./query-timeout" import { loadOptionalDriver } from "./resolve" import { emitBrowserSignIn, openInBrowser } from "./sign-in" import { emitReconnect, type ReconnectEvent } from "./reconnect-events" @@ -54,6 +56,21 @@ export function suppressSnowflakeLogging(snowflake: any): void { }) } +/** + * In assessment mode nothing but the warehouse may be contacted. When it loads, snowflake-sdk otherwise probes + * the AWS, Azure and GCP metadata and identity services to report the platform it runs on. + */ +export function keepSdkOffCloudMetadata(env: NodeJS.ProcessEnv = process.env): void { + const mode = env.ALTIMATE_ASSESSMENT_MODE?.trim().toLowerCase() + if (mode && mode !== "0" && mode !== "false") env.SNOWFLAKE_DISABLE_PLATFORM_DETECTION = "true" +} + +/** How long to wait for Snowflake to acknowledge a cancel before reporting the statement as possibly still running. */ +const CANCEL_WAIT_MS = 30_000 + +/** How a caller's `cancelAfterMs` reaches the statement it covers: why it was stopped, and the statement sent. */ +type StatementStop = { stopped?: "deadline" | "aborted"; statement?: any } + /** * Errors meaning the session is gone: 407002 is raised client-side for any * request on a terminated connection; 390111 (session no longer exists), @@ -183,10 +200,12 @@ function describeAccount(account: unknown): string { export async function connect( config: ConnectionConfig, sdk?: unknown, - seams?: { connectTimeoutMs?: number }, + seams?: { connectTimeoutMs?: number; cancelWaitMs?: number }, ): Promise { const connectTimeoutMs = seams?.connectTimeoutMs ?? CONNECT_TIMEOUT_MS + const cancelWaitMs = seams?.cancelWaitMs ?? CANCEL_WAIT_MS let snowflake: any + keepSdkOffCloudMetadata() snowflake = sdk ?? (await loadOptionalDriver("snowflake", "snowflake-sdk")) snowflake = snowflake.default || snowflake @@ -468,7 +487,11 @@ export async function connect( * reopened and a read-only statement is retried once; a write is not, because * it may already have run (see `isRetrySafe`). */ - async function executeQuery(sql: string, binds?: any[]): Promise<{ columns: string[]; rows: any[][] }> { + async function executeQuery( + sql: string, + binds?: any[], + stop?: StatementStop, + ): Promise<{ columns: string[]; rows: any[][] }> { // Captured before any wait: a statement queued behind a reconnect started on the old session's assumptions. const started = generation const startedInEpoch = epoch @@ -477,7 +500,7 @@ export async function connect( const used = connection const ranIn = generation try { - const result = await runTracked(used, sql, binds, ranIn) + const result = await runTracked(used, sql, binds, ranIn, stop) await noteSession(sql, binds, ranIn, startedInEpoch) return result } catch (err) { @@ -496,7 +519,7 @@ export async function connect( ) } const ranAgainIn = generation - const result = await runTracked(connection, sql, binds, ranAgainIn) + const result = await runTracked(connection, sql, binds, ranAgainIn, stop) await noteSession(sql, binds, ranAgainIn, startedInEpoch) return result } @@ -511,11 +534,11 @@ export async function connect( /** `runQuery`, counting a statement that may hold session state against the session it was sent to (`ran`) for * as long as it runs. Counted only once sent: one still waiting on a reconnect will run on the new session. */ - async function runTracked(conn: any, sql: string, binds: any[] | undefined, ran: number) { - if (!mayHoldState(sql, binds)) return runQuery(conn, sql, binds) + async function runTracked(conn: any, sql: string, binds: any[] | undefined, ran: number, stop?: StatementStop) { + if (!mayHoldState(sql, binds)) return runQuery(conn, sql, binds, stop) stateStatementsRunning.set(ran, (stateStatementsRunning.get(ran) ?? 0) + 1) try { - return await runQuery(conn, sql, binds) + return await runQuery(conn, sql, binds, stop) } finally { const left = (stateStatementsRunning.get(ran) ?? 1) - 1 if (left > 0) stateStatementsRunning.set(ran, left) @@ -523,8 +546,15 @@ export async function connect( } } - function runQuery(conn: any, sql: string, binds?: any[]): Promise<{ columns: string[]; rows: any[][] }> { + function runQuery( + conn: any, + sql: string, + binds?: any[], + stop?: StatementStop, + ): Promise<{ columns: string[]; rows: any[][] }> { return new Promise((resolve, reject) => { + // Stopped while waiting on a reconnect: the statement is never sent. + if (stop?.stopped) return reject(new Error("Snowflake statement stopped before it was sent")) const options: Record = { sqlText: sql, complete(err: Error | null, _stmt: any, rows: any[]) { @@ -541,10 +571,63 @@ export async function connect( }, } if (binds && binds.length > 0) options.binds = binds - conn.execute(options) + const statement = conn.execute(options) + if (stop) stop.statement = statement + }) + } + + function cancelStatement(statement: any): Promise { + return new Promise((resolve) => { + const timer = setTimeout(() => resolve(false), cancelWaitMs) + const finish = (cancelled: boolean) => { + clearTimeout(timer) + resolve(cancelled) + } + try { + statement.cancel((err: Error | null | undefined) => finish(!err)) + } catch { + finish(false) + } }) } + /** + * `cancelAfterMs` is an opt-in deadline: past it, or when `signal` aborts, the statement is cancelled on + * Snowflake. One still waiting on a reconnect is never sent, so nothing is left running. + */ + async function executeWithCancel( + sql: string, + binds: any[] | undefined, + deadlineMs: number, + signal?: AbortSignal, + ): Promise<{ columns: string[]; rows: any[][] }> { + signal?.throwIfAborted() + const stop: StatementStop = {} + const STOPPED = Symbol("stopped") + let wake: () => void = () => {} + const stopping = new Promise((resolve) => (wake = () => resolve(STOPPED))) + const halt = (why: "deadline" | "aborted") => { + stop.stopped ??= why + wake() + } + const timer = setTimeout(() => halt("deadline"), Math.min(deadlineMs, MAX_TIMER_MS)) + const onAbort = () => halt("aborted") + signal?.addEventListener("abort", onAbort, { once: true }) + const work = executeQuery(sql, binds, stop) + let outcome: { columns: string[]; rows: any[][] } | typeof STOPPED + try { + outcome = await Promise.race([work, stopping]) + } finally { + clearTimeout(timer) + signal?.removeEventListener("abort", onAbort) + } + if (outcome !== STOPPED) return outcome + work.catch(() => {}) + const cancelled = stop.statement ? await cancelStatement(stop.statement) : true + if (stop.stopped === "aborted") throw signal!.reason + throw new QueryTimeoutError({ engine: "snowflake", timeoutMs: deadlineMs, cancelled }) + } + return { async connect() { closed = false @@ -710,7 +793,11 @@ export async function connect( query = `${sql.replace(/;\s*$/, "")} LIMIT ${effectiveLimit + 1}` } - const result = await executeQuery(query, binds) + const deadlineMs = options?.cancelAfterMs + const result = + deadlineMs && deadlineMs > 0 + ? await executeWithCancel(query, binds, deadlineMs, options?.signal) + : await executeQuery(query, binds) const truncated = effectiveLimit > 0 && result.rows.length > effectiveLimit const rows = truncated ? result.rows.slice(0, effectiveLimit) diff --git a/packages/drivers/src/types.ts b/packages/drivers/src/types.ts index 00ec89cf00..e3046e0057 100644 --- a/packages/drivers/src/types.ts +++ b/packages/drivers/src/types.ts @@ -46,7 +46,7 @@ export interface ExecuteOptions { queryTimeoutMs?: number /** Cancel the statement on the server once this many milliseconds pass, or when * `signal` aborts. Opt-in: `executeQuery` fills in `queryTimeoutMs` for every - * call, so only callers that set this get the server-side cancel. Databricks only. */ + * call, so only callers that set this get the server-side cancel. Databricks and Snowflake. */ cancelAfterMs?: number /** DuckDB only: called once after handing this query to its helper; callback errors do not affect execution. */ onDispatch?: () => void diff --git a/packages/drivers/test/snowflake-deadline.test.ts b/packages/drivers/test/snowflake-deadline.test.ts new file mode 100644 index 0000000000..5c6e0dea9e --- /dev/null +++ b/packages/drivers/test/snowflake-deadline.test.ts @@ -0,0 +1,238 @@ +/** + * Snowflake server-side cancel: a caller-named `cancelAfterMs` cancels the statement when it runs + * out or when `signal` aborts; without one, execution is unchanged. + */ +import { beforeEach, describe, expect, test } from "bun:test" +import { connect, keepSdkOffCloudMetadata } from "../src/snowflake" +import { isQueryTimeoutError } from "../src/types" + +const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)) + +let finishAfterMs: number | undefined +let cancelError: Error | undefined +let cancelSilently = false +let queryError: Error | undefined +let cancels = 0 +let sent: string[] = [] +let up = true +let holdConnect: Promise | undefined +let failFirstWith: { code: string; message: string } | undefined + +beforeEach(() => { + finishAfterMs = undefined + cancelError = undefined + cancelSilently = false + queryError = undefined + cancels = 0 + sent = [] + up = true + holdConnect = undefined + failFirstWith = undefined +}) + +function fakeSdk() { + return { + configure() {}, + createConnection() { + return { + connect(cb: (err: Error | null) => void) { + if (holdConnect) holdConnect.then(() => cb(null)) + else cb(null) + }, + isUp: () => up, + execute(options: { sqlText: string; complete: (err: Error | null, stmt: unknown, rows: unknown[]) => void }) { + sent.push(options.sqlText) + if (failFirstWith) { + const failure = Object.assign(new Error(failFirstWith.message), { code: failFirstWith.code }) + failFirstWith = undefined + setTimeout(() => options.complete(failure, null, []), 1) + } else if (queryError) setTimeout(() => options.complete(queryError!, null, []), 1) + else if (finishAfterMs !== undefined) { + setTimeout(() => options.complete(null, null, [{ N: 1 }]), finishAfterMs) + } + return { + cancel(cb: (err?: Error) => void) { + cancels++ + if (!cancelSilently) cb(cancelError) + }, + } + }, + destroy(cb: (err: Error | null) => void) { + cb(null) + }, + } + }, + } +} + +async function connector(seams?: { cancelWaitMs?: number }) { + const one = await connect({ type: "snowflake", account: "a", user: "u", password: "p" } as never, fakeSdk(), seams) + await one.connect() + return one +} + +describe("Snowflake SDK in assessment mode", () => { + test("cloud metadata probes are switched off only when the mode is on", () => { + for (const value of ["true", "1", "yes"]) { + const env: NodeJS.ProcessEnv = { ALTIMATE_ASSESSMENT_MODE: value } + keepSdkOffCloudMetadata(env) + expect(env.SNOWFLAKE_DISABLE_PLATFORM_DETECTION).toBe("true") + } + // The SDK skips detection only on "true", so another value left in the environment is replaced. + const preset: NodeJS.ProcessEnv = { ALTIMATE_ASSESSMENT_MODE: "true", SNOWFLAKE_DISABLE_PLATFORM_DETECTION: "false" } + keepSdkOffCloudMetadata(preset) + expect(preset.SNOWFLAKE_DISABLE_PLATFORM_DETECTION).toBe("true") + for (const env of [{}, { ALTIMATE_ASSESSMENT_MODE: "" }, { ALTIMATE_ASSESSMENT_MODE: "0" }, { ALTIMATE_ASSESSMENT_MODE: "false" }] as NodeJS.ProcessEnv[]) { + keepSdkOffCloudMetadata(env) + expect(env.SNOWFLAKE_DISABLE_PLATFORM_DETECTION).toBeUndefined() + } + }) + + test("connecting applies it before the SDK is used", async () => { + const saved = { mode: process.env.ALTIMATE_ASSESSMENT_MODE, off: process.env.SNOWFLAKE_DISABLE_PLATFORM_DETECTION } + try { + process.env.ALTIMATE_ASSESSMENT_MODE = "true" + delete process.env.SNOWFLAKE_DISABLE_PLATFORM_DETECTION + await connect({ type: "snowflake", account: "a", user: "u", password: "p" } as never, fakeSdk()) + expect(process.env.SNOWFLAKE_DISABLE_PLATFORM_DETECTION).toBe("true") + } finally { + for (const [key, value] of [["ALTIMATE_ASSESSMENT_MODE", saved.mode], ["SNOWFLAKE_DISABLE_PLATFORM_DETECTION", saved.off]] as const) { + if (value === undefined) delete process.env[key] + else process.env[key] = value + } + } + }) +}) + +describe("Snowflake server-side cancel", () => { + test("a statement past `cancelAfterMs` is cancelled on Snowflake and reported as a timeout", async () => { + const db = await connector() + const error = await db.execute("SELECT 1", 10, [], { cancelAfterMs: 30 }).catch((e) => e) + expect(isQueryTimeoutError(error)).toBe(true) + expect(error.cancelled).toBe(true) + expect(error.timeoutMs).toBe(30) + expect(error.engine).toBe("snowflake") + expect(cancels).toBe(1) + }) + + test("`queryTimeoutMs` alone never cancels: a slow statement still returns its rows", async () => { + finishAfterMs = 80 + const db = await connector() + const result = await db.execute("SELECT 1", 10, [], { queryTimeoutMs: 30 }) + expect(result.rows).toEqual([[1]]) + expect(cancels).toBe(0) + }) + + test("an aborted signal cancels the statement and rejects with the abort reason", async () => { + const controller = new AbortController() + const db = await connector() + const pending = db.execute("SELECT 1", 10, [], { cancelAfterMs: 60_000, signal: controller.signal }).catch((e) => e) + await sleep(10) + controller.abort(new Error("session stopped")) + const error = await pending + expect(isQueryTimeoutError(error)).toBe(false) + expect(error.message).toBe("session stopped") + expect(cancels).toBe(1) + }) + + test("an already-aborted signal sends nothing", async () => { + const controller = new AbortController() + controller.abort(new Error("session stopped")) + const db = await connector() + const error = await db.execute("SELECT 1", 10, [], { cancelAfterMs: 1_000, signal: controller.signal }).catch((e) => e) + expect(error.message).toBe("session stopped") + expect(sent).toEqual([]) + }) + + test("a statement still waiting on a reconnect at the deadline is never sent", async () => { + const db = await connector() + let release: () => void = () => {} + holdConnect = new Promise((resolve) => (release = resolve)) + up = false + const error = await db.execute("SELECT 1", 10, [], { cancelAfterMs: 30 }).catch((e) => e) + expect(isQueryTimeoutError(error)).toBe(true) + // Nothing reached Snowflake, so nothing is left running there. + expect(error.cancelled).toBe(true) + expect(cancels).toBe(0) + up = true + release() + await sleep(20) + expect(sent).toEqual([]) + }) + + test("when the cancel fails, the error says the statement may still be running", async () => { + cancelError = new Error("network") + const db = await connector() + const error = await db.execute("SELECT 1", 10, [], { cancelAfterMs: 30 }).catch((e) => e) + expect(isQueryTimeoutError(error)).toBe(true) + expect(error.cancelled).toBe(false) + expect(error.message).toContain("it may still be running on the server") + }) + + test("when Snowflake never answers the cancel, the error says the statement may still be running", async () => { + cancelSilently = true + const db = await connector({ cancelWaitMs: 10 }) + const error = await db.execute("SELECT 1", 10, [], { cancelAfterMs: 20 }).catch((e) => e) + expect(isQueryTimeoutError(error)).toBe(true) + expect(error.cancelled).toBe(false) + expect(cancels).toBe(1) + }) + + test("a read retried after the session closed is still covered by the deadline", async () => { + failFirstWith = { code: "407002", message: "Unable to perform operation using terminated connection." } + const db = await connector() + const error = await db.execute("SELECT 1", 10, [], { cancelAfterMs: 40 }).catch((e) => e) + expect(isQueryTimeoutError(error)).toBe(true) + expect(sent).toEqual(["SELECT 1 LIMIT 11", "SELECT 1 LIMIT 11"]) + expect(cancels).toBe(1) + }) + + test("a statement whose session closed may still be running, so the deadline tries to cancel it", async () => { + // A closed-session error can arrive while the statement keeps running on Snowflake. + failFirstWith = { code: "407002", message: "Unable to perform operation using terminated connection." } + cancelError = new Error("session is gone") + const db = await connector() + let release: () => void = () => {} + holdConnect = new Promise((resolve) => (release = resolve)) + const error = await db.execute("SELECT 1", 10, [], { cancelAfterMs: 30 }).catch((e) => e) + expect(isQueryTimeoutError(error)).toBe(true) + expect(error.cancelled).toBe(false) + expect(cancels).toBe(1) + release() + await sleep(10) + // The retry waiting on the reconnect is never sent. + expect(sent).toEqual(["SELECT 1 LIMIT 11"]) + }) + + test("the abort listener does not outlive the statement", async () => { + finishAfterMs = 1 + const controller = new AbortController() + const added: unknown[] = [] + const removed: unknown[] = [] + const add = controller.signal.addEventListener.bind(controller.signal) + const remove = controller.signal.removeEventListener.bind(controller.signal) + controller.signal.addEventListener = ((type: string, fn: never, options: never) => (added.push(fn), add(type, fn, options))) as never + controller.signal.removeEventListener = ((type: string, fn: never) => (removed.push(fn), remove(type, fn))) as never + const db = await connector() + await db.execute("SELECT 1", 10, [], { cancelAfterMs: 5_000, signal: controller.signal }) + expect(added.length).toBe(1) + expect(removed).toEqual(added) + }) + + test("a statement that finishes in time returns its rows and is not cancelled", async () => { + finishAfterMs = 1 + const db = await connector() + const result = await db.execute("SELECT 1", 10, [], { cancelAfterMs: 5_000 }) + expect(result.rows).toEqual([[1]]) + expect(cancels).toBe(0) + }) + + test("a query error inside the deadline is the query's own error, not a timeout", async () => { + queryError = new Error("SQL compilation error") + const db = await connector() + const error = await db.execute("SELECT 1", 10, [], { cancelAfterMs: 5_000 }).catch((e) => e) + expect(isQueryTimeoutError(error)).toBe(false) + expect(error.message).toContain("SQL compilation error") + expect(cancels).toBe(0) + }) +}) From db06540439bc75414dae8248c6bf7a2cea42e269 Mon Sep 17 00:00:00 2001 From: Shreyas Telkar Date: Wed, 18 Nov 2026 14:49:31 -0800 Subject: [PATCH 25/34] feat: `finops_snowflake_savings`, the Snowflake warehouse savings check Measures warehouse usage from `SHOW WAREHOUSES` and `ACCOUNT_USAGE`, read-only, and hands the facts and the price of a credit to altimate-core. It runs only with `confirm: true` and stops after 9 minutes. Assessment mode and the zero-egress suite cover it. Co-Authored-By: Claude Opus 5.5 --- .../data-engineering/tools/finops-tools.md | 31 + packages/opencode/src/agent/agent.ts | 1 + .../src/altimate/native/finops/register.ts | 7 + .../native/finops/snowflake-savings.ts | 563 ++++++++++++++ .../opencode/src/altimate/native/types.ts | 59 ++ .../src/altimate/tool-cli/allow-list.ts | 2 + .../tools/finops-databricks-savings.ts | 4 +- .../tools/finops-snowflake-savings.ts | 172 +++++ packages/opencode/src/tool/registry.ts | 2 + .../test/agent/analyst-data-explorer.test.ts | 1 + .../altimate/assessment-zero-egress.test.ts | 53 +- .../altimate/finops-snowflake-savings.test.ts | 707 ++++++++++++++++++ 12 files changed, 1599 insertions(+), 3 deletions(-) create mode 100644 packages/opencode/src/altimate/native/finops/snowflake-savings.ts create mode 100644 packages/opencode/src/altimate/tools/finops-snowflake-savings.ts create mode 100644 packages/opencode/test/altimate/finops-snowflake-savings.test.ts diff --git a/docs/docs/data-engineering/tools/finops-tools.md b/docs/docs/data-engineering/tools/finops-tools.md index b984c10c7e..ac07b529b4 100644 --- a/docs/docs/data-engineering/tools/finops-tools.md +++ b/docs/docs/data-engineering/tools/finops-tools.md @@ -206,6 +206,37 @@ Classic (sql warehouse, workspace 525779153077873) · Migrate Classic/Pro to Ser $29.29 in DBUs over the window; serverless would bill $4.44 for the busy time alone ``` + +--- + +## finops_snowflake_savings + +Check a Snowflake account for savings with Altimate's rules. It lists warehouses with `SHOW WAREHOUSES` and reads seven days of `SNOWFLAKE.ACCOUNT_USAGE` (`WAREHOUSE_METERING_HISTORY`, `WAREHOUSE_EVENTS_HISTORY`, `QUERY_HISTORY`), read-only, on the connection's warehouse. The role needs `IMPORTED PRIVILEGES` on the `SNOWFLAKE` database, and sees only the warehouses it has access to; warehouses that billed credits but cannot be listed are named as not checked. + +- **High Idle Time:** on days a warehouse sat idle for more than 40% of its active time, the idle share of that day's cost. +- **Downsize Warehouse:** nearly all queries finish within 5 seconds, or nearly all active periods last 80 seconds or less, so one size smaller would do. +- **Underutilized Cluster:** the highest cluster of a multi-cluster warehouse is barely needed. It is reported without a dollar estimate. +- **Frequent Restart Overcharge:** a warehouse that suspends and resumes within one billed minute pays the overlap twice. + +The tool measures each warehouse's usage; altimate-core decides what qualifies and what it saves. Credits are priced from `ORGANIZATION_USAGE.RATE_SHEET_DAILY` when the role can read it in dollars, otherwise at $3.00 (also for an account billed in another currency), and the output says which. Each warehouse is counted once in the total. + +**What it does not assess.** These are four of Altimate's warehouse checks. Storage, serverless features and individual query tuning are not covered. A warehouse that never suspended in the window has no completed active period, so it gets no idle finding. Days are UTC, and the window ends at 00:00 UTC on the latest day at least six hours past, because `ACCOUNT_USAGE` lags by up to that long. + +**Asks first, and stops long queries.** The check runs only when called with `confirm: true`; without it, the agent is told to ask you first, since a run can resume a suspended warehouse that then bills. It runs six queries and stops after 9 minutes, connecting included, cancelling any query still running on Snowflake (or sooner, at an `altimate run --time-budget` cutoff); stopping the session cancels the running query too. + +**Assessment mode.** `ALTIMATE_ASSESSMENT_MODE=true` works as it does for `finops_databricks_savings`: the run contacts your Snowflake account, and turns off the Snowflake driver's cloud-platform detection. If a result is large enough for Snowflake to deliver it from its own cloud storage, the driver fetches it from there too. + +``` +> finops_snowflake_savings --warehouse prod_sf --confirm + +Snowflake savings: 12 warehouses checked, the 7 days to 2026-11-18, at $3.00 a credit from the account's rate sheet. +Total: $54.00–$90.00 a month on 1 warehouse, each counted once at its largest saving. + +BI_WH (warehouse) · Downsize Warehouse · $54.00–$90.00 a month (medium) + Large could be Medium: 95.0% of its active periods lasted 80 seconds or less + The saving is a share of $42.00 over the window +``` + --- ## finops_role_grants diff --git a/packages/opencode/src/agent/agent.ts b/packages/opencode/src/agent/agent.ts index 2ac7aec2a7..100566f899 100644 --- a/packages/opencode/src/agent/agent.ts +++ b/packages/opencode/src/agent/agent.ts @@ -268,6 +268,7 @@ export const layer = Layer.effect( finops_expensive_queries: "allow", finops_warehouse_advice: "allow", finops_databricks_savings: "allow", + finops_snowflake_savings: "allow", finops_unused_resources: "allow", finops_role_grants: "allow", finops_role_hierarchy: "allow", diff --git a/packages/opencode/src/altimate/native/finops/register.ts b/packages/opencode/src/altimate/native/finops/register.ts index 43d4a263d5..bdefd31e49 100644 --- a/packages/opencode/src/altimate/native/finops/register.ts +++ b/packages/opencode/src/altimate/native/finops/register.ts @@ -7,6 +7,7 @@ import { getQueryHistory } from "./query-history" import { analyzeCredits, getExpensiveQueries } from "./credit-analyzer" import { adviseWarehouse } from "./warehouse-advisor" import { databricksSavings } from "./databricks-savings" +import { snowflakeSavings } from "./snowflake-savings" import { findUnusedResources } from "./unused-resources" import { queryGrants, queryRoleHierarchy, queryUserRoles } from "./role-access" import type { @@ -20,6 +21,8 @@ import type { WarehouseAdvisorResult, DatabricksSavingsParams, DatabricksSavingsResult, + SnowflakeSavingsParams, + SnowflakeSavingsResult, UnusedResourcesParams, UnusedResourcesResult, RoleGrantsParams, @@ -53,6 +56,10 @@ register("finops.databricks_savings", async (params: DatabricksSavingsParams, co return databricksSavings(params, undefined, context) }) +register("finops.snowflake_savings", async (params: SnowflakeSavingsParams, context): Promise => { + return snowflakeSavings(params, undefined, context) +}) + register("finops.unused_resources", async (params: UnusedResourcesParams): Promise => { return findUnusedResources(params) }) diff --git a/packages/opencode/src/altimate/native/finops/snowflake-savings.ts b/packages/opencode/src/altimate/native/finops/snowflake-savings.ts new file mode 100644 index 0000000000..e703db1ead --- /dev/null +++ b/packages/opencode/src/altimate/native/finops/snowflake-savings.ts @@ -0,0 +1,563 @@ +// altimate_change - new file +/** + * Snowflake savings: measures usage facts per warehouse from `SHOW WAREHOUSES` and + * `SNOWFLAKE.ACCOUNT_USAGE`, and hands them to altimate-core, which decides what qualifies and what + * that saves. The rules live in core; this file only measures. + * + * Days are UTC. Every window ends at 00:00 UTC on the window's end date, and a run happens at least + * six hours after that, past the latency of each view read here: WAREHOUSE_METERING_HISTORY (three + * hours, six for cloud services credits), WAREHOUSE_EVENTS_HISTORY (three) and QUERY_HISTORY (45 + * minutes). + */ + +import * as core from "@altimateai/altimate-core" +import { isQueryTimeoutError } from "@altimateai/drivers/types" +import { Flag } from "@opencode-ai/core/flag/flag" +import * as Registry from "../connections/registry" +import type { QueryContext } from "../query-execution" +import { RunBudget } from "../../run-budget" +import { savingsWindow } from "./databricks-savings" +import { resolveFinopsWarehouse } from "./warehouse-resolver" +import type { SnowflakeSavingsParams, SnowflakeSavingsReport, SnowflakeSavingsResult } from "../types" + +const SUPPORTED_TYPES = ["snowflake"] as const + +/** Days of usage the facts cover; the rules price a day of this window. */ +const LOOKBACK_DAYS = 7 + +/** Rows read per query; a query that fills it says so in the result. */ +export const ROW_LIMIT = 100_000 + +/** The driver asks for one row more than its limit, and `SHOW` accepts a `LIMIT` of at most 10,000. */ +export const SHOW_ROW_LIMIT = 9_999 + +/** Dollars per credit when the account's rate sheet cannot be read: Enterprise list price. */ +export const LIST_PRICE_PER_CREDIT = 3 + +// The whole check stops after this long, inside the 10 minutes a Claude Code plugin's process may +// run, so a running query is cancelled here and not left behind when the process is killed. +export const SAVINGS_RUN_LIMIT_MS = 9 * 60_000 + +/** The end of the window as a timestamp for filtering `ACCOUNT_USAGE`, and as UTC wall-clock time. */ +const WINDOW_END = `TO_TIMESTAMP_TZ(? || ' 00:00:00 +0000', 'YYYY-MM-DD HH24:MI:SS TZHTZM')` +const WINDOW_END_UTC = `TO_TIMESTAMP_NTZ(?, 'YYYY-MM-DD')` + +export const SNOWFLAKE_WAREHOUSES_SQL = `SHOW WAREHOUSES` + +/** + * Credits per warehouse by UTC day (`usage_day`), and by the 24 hours from 23:00 UTC the day before + * (`idle_day`), the window the idle check reads. + */ +export const SNOWFLAKE_METERING_FACTS_SQL = ` +WITH m AS ( + SELECT WAREHOUSE_ID AS warehouse_id, WAREHOUSE_NAME AS warehouse_name, + TO_TIMESTAMP_NTZ(CONVERT_TIMEZONE('UTC', START_TIME)) AS hour_start, CREDITS_USED AS credits_used + FROM SNOWFLAKE.ACCOUNT_USAGE.WAREHOUSE_METERING_HISTORY + WHERE START_TIME >= DATEADD(hour, -169, ${WINDOW_END}) AND START_TIME < ${WINDOW_END} AND WAREHOUSE_ID > 0 +) +SELECT warehouse_id, warehouse_name, + IFF(hour_start >= DATEADD(day, -7, ${WINDOW_END_UTC}), TO_VARCHAR(hour_start::DATE, 'YYYY-MM-DD'), NULL) AS usage_day, + IFF(hour_start < DATEADD(hour, -1, ${WINDOW_END_UTC}), TO_VARCHAR(DATEADD(hour, 1, hour_start)::DATE, 'YYYY-MM-DD'), NULL) AS idle_day, + SUM(credits_used) AS credits_used +FROM m GROUP BY 1, 2, 3, 4 ORDER BY 2, 3, 4 +` + +/** + * Per warehouse, cluster and day: the periods from each cluster's resume to its next event that ran at + * least one query, their seconds, and the seconds with no query running. Overlapping queries count as + * one stretch of work. Seconds are `DATEDIFF('second', ...)`, which counts second boundaries crossed: + * this is how the idle times stored by Altimate's hosted product are counted. + */ +export const SNOWFLAKE_PERIOD_FACTS_SQL = ` +WITH ev AS ( + SELECT WAREHOUSE_ID AS warehouse_id, WAREHOUSE_NAME AS warehouse_name, CLUSTER_NUMBER AS cluster_number, + TO_TIMESTAMP_NTZ(CONVERT_TIMEZONE('UTC', TIMESTAMP)) AS ts, EVENT_NAME AS event_name + FROM SNOWFLAKE.ACCOUNT_USAGE.WAREHOUSE_EVENTS_HISTORY + WHERE TIMESTAMP >= DATEADD(day, -9, ${WINDOW_END}) AND TIMESTAMP < DATEADD(day, 1, ${WINDOW_END}) + AND EVENT_NAME IN ('RESUME_CLUSTER', 'SUSPEND_CLUSTER') AND EVENT_STATE = 'STARTED' +), +status AS ( + SELECT warehouse_id, warehouse_name, cluster_number, ts AS valid_from, event_name, + LEAD(ts) OVER (PARTITION BY warehouse_id, cluster_number ORDER BY ts) AS valid_to + FROM ev +), +periods AS ( + SELECT warehouse_id, warehouse_name, cluster_number, valid_from, valid_to + FROM status + WHERE event_name = 'RESUME_CLUSTER' AND valid_to IS NOT NULL + AND DATEDIFF('millisecond', valid_from, valid_to) >= 1000 + AND valid_from >= DATEADD(hour, -169, ${WINDOW_END_UTC}) AND valid_from < ${WINDOW_END_UTC} +), +q AS ( + SELECT WAREHOUSE_ID AS warehouse_id, CLUSTER_NUMBER AS cluster_number, + TO_TIMESTAMP_NTZ(CONVERT_TIMEZONE('UTC', START_TIME)) AS st, + TO_TIMESTAMP_NTZ(CONVERT_TIMEZONE('UTC', END_TIME)) AS et + FROM SNOWFLAKE.ACCOUNT_USAGE.QUERY_HISTORY + WHERE START_TIME >= DATEADD(day, -10, ${WINDOW_END}) AND START_TIME < DATEADD(day, 1, ${WINDOW_END}) + AND WAREHOUSE_ID IS NOT NULL AND WAREHOUSE_SIZE IS NOT NULL AND CLUSTER_NUMBER IS NOT NULL +), +adjusted AS ( + SELECT p.warehouse_id, p.warehouse_name, p.cluster_number, p.valid_from, p.valid_to, + GREATEST(q.st, p.valid_from) AS query_start, LEAST(q.et, p.valid_to) AS query_end + FROM periods p JOIN q + ON q.warehouse_id = p.warehouse_id AND q.cluster_number = p.cluster_number + AND q.st < p.valid_to AND q.et > p.valid_from +), +flagged AS ( + SELECT *, IFF(query_start > MAX(query_end) OVER ( + PARTITION BY warehouse_id, cluster_number, valid_from ORDER BY query_start, query_end + ROWS BETWEEN UNBOUNDED PRECEDING AND 1 PRECEDING), 1, 0) AS later_group, + ROW_NUMBER() OVER (PARTITION BY warehouse_id, cluster_number, valid_from ORDER BY query_start, query_end) AS rn + FROM adjusted +), +grouped AS ( + SELECT *, SUM(IFF(rn = 1, 1, later_group)) OVER ( + PARTITION BY warehouse_id, cluster_number, valid_from ORDER BY rn + ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS group_id + FROM flagged +), +windows AS ( + SELECT warehouse_id, warehouse_name, cluster_number, valid_from, valid_to, group_id, + MIN(query_start) AS group_start, MAX(query_end) AS group_end + FROM grouped GROUP BY 1, 2, 3, 4, 5, 6 +), +gaps AS ( + SELECT *, LAG(group_end) OVER (PARTITION BY warehouse_id, cluster_number, valid_from ORDER BY group_start) AS prev_end, + LEAD(group_start) OVER (PARTITION BY warehouse_id, cluster_number, valid_from ORDER BY group_start) AS next_start + FROM windows +), +per_period AS ( + SELECT warehouse_id, warehouse_name, cluster_number, valid_from, + DATEDIFF('second', valid_from, valid_to) AS active_seconds, + SUM(DATEDIFF('second', COALESCE(prev_end, valid_from), group_start) + + IFF(next_start IS NULL, DATEDIFF('second', group_end, valid_to), 0)) AS idle_seconds + FROM gaps GROUP BY 1, 2, 3, 4, 5 +) +SELECT warehouse_id, warehouse_name, cluster_number, + IFF(valid_from >= DATEADD(day, -7, ${WINDOW_END_UTC}), TO_VARCHAR(valid_from::DATE, 'YYYY-MM-DD'), NULL) AS usage_day, + IFF(valid_from < DATEADD(hour, -1, ${WINDOW_END_UTC}), TO_VARCHAR(DATEADD(hour, 1, valid_from)::DATE, 'YYYY-MM-DD'), NULL) AS idle_day, + COUNT(*) AS periods, COUNT_IF(active_seconds <= 80) AS short_periods, + SUM(active_seconds) AS active_seconds, SUM(idle_seconds) AS idle_seconds +FROM per_period GROUP BY 1, 2, 3, 4, 5 ORDER BY 2, 3, 4, 5 +` + +/** + * Per warehouse and day: how many times it resumed, suspended and resumed again less than a minute after + * the first resume, by the seconds between the two resumes, counted as for active periods. The event log + * repeats each warehouse-level event, so events are made distinct first. + */ +export const SNOWFLAKE_RESTART_FACTS_SQL = ` +WITH e AS ( + SELECT DISTINCT WAREHOUSE_ID AS warehouse_id, WAREHOUSE_NAME AS warehouse_name, + TO_TIMESTAMP_NTZ(CONVERT_TIMEZONE('UTC', TIMESTAMP)) AS ts, EVENT_NAME AS event_name + FROM SNOWFLAKE.ACCOUNT_USAGE.WAREHOUSE_EVENTS_HISTORY + WHERE TIMESTAMP >= DATEADD(day, -7, ${WINDOW_END}) AND TIMESTAMP < ${WINDOW_END} + AND EVENT_NAME IN ('RESUME_WAREHOUSE', 'SUSPEND_WAREHOUSE', 'WAREHOUSE_CONSISTENT') +), +s AS ( + SELECT warehouse_id, warehouse_name, ts, event_name, + LEAD(event_name, 1) OVER (PARTITION BY warehouse_id, ts::DATE ORDER BY ts, event_name) AS n1, + LEAD(event_name, 2) OVER (PARTITION BY warehouse_id, ts::DATE ORDER BY ts, event_name) AS n2, + LEAD(event_name, 3) OVER (PARTITION BY warehouse_id, ts::DATE ORDER BY ts, event_name) AS n3, + LEAD(event_name, 4) OVER (PARTITION BY warehouse_id, ts::DATE ORDER BY ts, event_name) AS n4, + LEAD(ts, 4) OVER (PARTITION BY warehouse_id, ts::DATE ORDER BY ts, event_name) AS t4 + FROM e +) +SELECT warehouse_id, warehouse_name, TO_VARCHAR(ts::DATE, 'YYYY-MM-DD') AS day, + DATEDIFF('second', ts, t4) AS gap_seconds, COUNT(*) AS bounces +FROM s +WHERE event_name = 'RESUME_WAREHOUSE' AND n1 = 'WAREHOUSE_CONSISTENT' AND n2 = 'SUSPEND_WAREHOUSE' + AND n3 = 'WAREHOUSE_CONSISTENT' AND n4 = 'RESUME_WAREHOUSE' + AND DATEDIFF('second', ts, t4) < 60 +GROUP BY 1, 2, 3, 4 ORDER BY 2, 3, 4 +` + +/** + * Per warehouse: queries that ended in the window, those that executed for five seconds or less, and the + * credits attributed to those: each metered hour's compute credits shared by execution time, plus the + * query's share of the day's billable cloud services credits. + */ +export const SNOWFLAKE_QUERY_FACTS_SQL = ` +WITH metered AS ( + SELECT WAREHOUSE_ID AS warehouse_id, TO_TIMESTAMP_NTZ(CONVERT_TIMEZONE('UTC', START_TIME)) AS hour_start, + CREDITS_USED_COMPUTE AS compute_credits, CREDITS_USED_CLOUD_SERVICES AS cloud_credits + FROM SNOWFLAKE.ACCOUNT_USAGE.WAREHOUSE_METERING_HISTORY + WHERE START_TIME >= DATEADD(day, -9, ${WINDOW_END}) AND START_TIME < ${WINDOW_END} +), +hourly AS (SELECT warehouse_id, hour_start, SUM(compute_credits) AS compute_credits FROM metered WHERE warehouse_id > 0 GROUP BY 1, 2), +daily AS ( + SELECT hour_start::DATE AS day, SUM(cloud_credits) AS cloud_credits, + GREATEST(SUM(cloud_credits) - SUM(compute_credits) * 0.1, 0) AS billable_cloud_credits + FROM metered GROUP BY 1 +), +q AS ( + SELECT QUERY_ID AS query_id, WAREHOUSE_ID AS warehouse_id, WAREHOUSE_NAME AS warehouse_name, WAREHOUSE_SIZE AS warehouse_size, + EXECUTION_TIME AS execution_ms, CREDITS_USED_CLOUD_SERVICES AS cloud_credits, + TO_TIMESTAMP_NTZ(CONVERT_TIMEZONE('UTC', START_TIME)) AS st, + TO_TIMESTAMP_NTZ(CONVERT_TIMEZONE('UTC', END_TIME)) AS et, + DATEADD('millisecond', COALESCE(QUEUED_OVERLOAD_TIME, 0) + COALESCE(COMPILATION_TIME, 0) + COALESCE(QUEUED_PROVISIONING_TIME, 0) + + COALESCE(QUEUED_REPAIR_TIME, 0) + COALESCE(LIST_EXTERNAL_FILES_TIME, 0), + TO_TIMESTAMP_NTZ(CONVERT_TIMEZONE('UTC', START_TIME))) AS exec_start + FROM SNOWFLAKE.ACCOUNT_USAGE.QUERY_HISTORY + WHERE START_TIME >= DATEADD(day, -9, ${WINDOW_END}) AND START_TIME < ${WINDOW_END} AND WAREHOUSE_ID IS NOT NULL +), +overlap AS ( + SELECT q.query_id, q.warehouse_id, h.hour_start, h.compute_credits, + DATEDIFF('millisecond', GREATEST(q.exec_start, h.hour_start), LEAST(q.et, DATEADD(hour, 1, h.hour_start))) AS ran_ms + FROM q JOIN hourly h + ON h.warehouse_id = q.warehouse_id AND h.hour_start >= DATE_TRUNC('hour', q.exec_start) AND h.hour_start < q.et + WHERE q.warehouse_size IS NOT NULL AND q.et > q.st +), +allocated AS ( + SELECT query_id, SUM(compute_credits * ran_ms / total_ms) AS compute_credits + FROM (SELECT *, SUM(ran_ms) OVER (PARTITION BY warehouse_id, hour_start) AS total_ms FROM overlap WHERE ran_ms > 0) + GROUP BY 1 +) +SELECT q.warehouse_id, q.warehouse_name, + COUNT(*) AS query_count, + COUNT_IF(q.execution_ms <= 5000) AS sub_five_second_queries, + COALESCE(SUM(IFF(q.execution_ms <= 5000, + COALESCE(a.compute_credits, 0) + + IFF(d.cloud_credits > 0, q.cloud_credits / d.cloud_credits * d.billable_cloud_credits, 0), 0)), 0) AS sub_five_second_credits +FROM q +LEFT JOIN allocated a ON a.query_id = q.query_id +LEFT JOIN daily d ON d.day = q.st::DATE +WHERE q.et >= DATEADD(day, -7, ${WINDOW_END_UTC}) AND q.et < ${WINDOW_END_UTC} +GROUP BY 1, 2 ORDER BY 2 +` + +/** The account's contracted price of a warehouse credit, latest before the window's end. */ +export const SNOWFLAKE_RATE_SQL = ` +SELECT EFFECTIVE_RATE AS effective_rate, CURRENCY AS currency, TO_VARCHAR(DATE, 'YYYY-MM-DD') AS rate_date +FROM SNOWFLAKE.ORGANIZATION_USAGE.RATE_SHEET_DAILY +WHERE ACCOUNT_NAME = CURRENT_ACCOUNT_NAME() AND SERVICE_TYPE = 'WAREHOUSE_METERING' AND USAGE_TYPE = 'compute' AND DATE < ${WINDOW_END_UTC}::DATE +ORDER BY DATE DESC LIMIT 1 +` + +/** Core's rules: a `SnowflakeSavingsInput` as JSON in, the report out. */ +export type RulesEngine = (input: string) => SnowflakeSavingsReport + +type CoreModule = { snowflakeSavings?: unknown; initSdk?: (...args: [string, string, string, boolean, boolean]) => void } + +/** The installed core's rules, or null when its binary predates them. See `databricks-savings.ts` for assessment mode. */ +export function rulesEngine(lib: CoreModule = core as unknown as CoreModule): RulesEngine | null { + const found = lib.snowflakeSavings + if (typeof found !== "function") return null + if (Flag.ALTIMATE_ASSESSMENT_MODE) lib.initSdk?.("", "", "", false, false) + return found as RulesEngine +} + +type Row = Record + +function num(value: unknown): number { + const parsed = typeof value === "bigint" ? Number(value) : Number(value ?? 0) + return Number.isFinite(parsed) ? parsed : 0 +} + +const whole = (value: unknown) => Math.max(0, Math.round(num(value))) + +function optionalWhole(value: unknown): number | null { + return value == null || value === "" ? null : whole(value) +} + +function records(result: { columns: string[]; rows: unknown[][] }): Row[] { + return result.rows.map((row) => Object.fromEntries(result.columns.map((name, at) => [name, row[at]]))) +} + +/** A map entry, created on first use. */ +function entry(map: Map, key: string, create: () => V): V { + let found = map.get(key) + if (found === undefined) { + found = create() + map.set(key, found) + } + return found +} + +export interface FactRows { + warehouses: Row[] + metering: Row[] + periods: Row[] + restarts: Row[] + queries: Row[] +} + +/** Milliseconds of a `created_on` value, or NaN when it cannot be read. */ +function timeOf(value: unknown): number { + if (value instanceof Date) return value.getTime() + const json = (value as { toJSON?: () => unknown } | null | undefined)?.toJSON?.() + return Date.parse(String(json ?? value ?? "")) +} + +/** + * The input core reads, one entry per warehouse `SHOW WAREHOUSES` lists, which gives names and no ids. + * Usage belongs to a warehouse id: a renamed warehouse keeps its id, so an id's usage goes to the last + * name it was seen under. A dropped warehouse's name can come back on a new one, so a name keeps its + * newest id, and usage that ended before the listed warehouse was created is its predecessor's. + * `unlisted` names the warehouses that billed credits but are not listed: dropped, or hidden from the role. + */ +export function buildFacts(rows: FactRows) { + const listed = new Set(rows.warehouses.map((row) => String(row.name ?? ""))) + const lastSeen = new Map() + const see = (row: Row, day: unknown) => { + const id = num(row.warehouse_id) + const name = String(row.warehouse_name ?? "") + const at = String(day ?? "") + const known = lastSeen.get(id) + if (!known || at > known.day || (at === known.day && listed.has(name) && !listed.has(known.name))) { + lastSeen.set(id, { day: at, name }) + } + } + for (const row of [...rows.metering, ...rows.periods]) see(row, row.usage_day ?? row.idle_day) + for (const row of rows.restarts) see(row, row.day) + for (const row of rows.queries) see(row, "") + + const newestId = new Map() + for (const [id, { name }] of lastSeen) newestId.set(name, Math.max(newestId.get(name) ?? 0, id)) + const nameOf = (row: Row) => lastSeen.get(num(row.warehouse_id))?.name ?? "" + const of = (list: Row[]) => { + const byName = new Map() + for (const row of list) { + const name = nameOf(row) + if (newestId.get(name) === num(row.warehouse_id)) entry(byName, name, () => []).push(row) + } + return byName + } + const metering = of(rows.metering) + const periods = of(rows.periods) + const restarts = of(rows.restarts) + const queries = of(rows.queries) + + /** The listed warehouse's id, unless its usage all predates it. */ + const idFor = (row: Row): number | undefined => { + const id = newestId.get(String(row.name ?? "")) + if (id === undefined) return undefined + const created = timeOf(row.created_on) + const day = lastSeen.get(id)?.day + if (day && Number.isFinite(created) && day < new Date(created).toISOString().slice(0, 10)) return undefined + return id + } + + const facts = rows.warehouses.map((row) => { + const name = String(row.name ?? "") + const id = idFor(row) + const usedBy = (byName: Map) => (id === undefined ? [] : (byName.get(name) ?? [])) + const usage = new Map() + const idle = new Map() + const clusters = new Map>() + const usageDay = (day: string) => entry(usage, day, () => ({ day, credits_used: null, active_periods: 0, short_periods: 0 })) + const idleDay = (day: string) => entry(idle, day, () => ({ day, active_seconds: 0, idle_seconds: 0, credits_used: 0 })) + + for (const one of usedBy(metering)) { + const credits = num(one.credits_used) + if (one.usage_day != null) { + const day = usageDay(String(one.usage_day)) + day.credits_used = (day.credits_used ?? 0) + credits + } + if (one.idle_day != null) idleDay(String(one.idle_day)).credits_used += credits + } + for (const one of usedBy(periods)) { + const active = whole(one.active_seconds) + const idleSeconds = whole(one.idle_seconds) + if (one.usage_day != null) { + const key = String(one.usage_day) + const day = usageDay(key) + day.active_periods += whole(one.periods) + day.short_periods += whole(one.short_periods) + const number = whole(one.cluster_number) + const cluster = entry(entry(clusters, key, () => new Map()), String(number), () => ({ + cluster_number: number, + idle_seconds: 0, + busy_seconds: 0, + })) + cluster.idle_seconds += idleSeconds + cluster.busy_seconds += active - idleSeconds + } + if (one.idle_day != null) { + const day = idleDay(String(one.idle_day)) + day.active_seconds += active + day.idle_seconds += idleSeconds + } + } + // A rename splits a day's or the window's rows by name; they are one warehouse's. + const gaps = new Map>() + for (const one of usedBy(restarts)) { + const seconds = whole(one.gap_seconds) + const gap = entry(entry(gaps, String(one.day), () => new Map()), String(seconds), () => ({ gap_seconds: seconds, bounces: 0 })) + gap.bounces += whole(one.bounces) + } + const query = { count: 0, short: 0, credits: 0 } + for (const one of usedBy(queries)) { + query.count += whole(one.query_count) + query.short += whole(one.sub_five_second_queries) + query.credits += num(one.sub_five_second_credits) + } + return { + warehouse_id: id ? String(id) : name, + warehouse_name: name, + warehouse_size: row.size == null ? null : String(row.size), + warehouse_type: row.type == null ? null : String(row.type), + min_cluster_count: optionalWhole(row.min_cluster_count), + max_cluster_count: optionalWhole(row.max_cluster_count), + query_count: query.count, + sub_five_second_queries: query.short, + sub_five_second_credits: query.credits, + usage_days: [...usage.values()], + idle_days: [...idle.values()], + cluster_days: [...clusters].map(([day, byNumber]) => ({ day, clusters: [...byNumber.values()] })), + restart_days: [...gaps].map(([day, byGap]) => ({ day, gaps: [...byGap.values()] })), + } + }) + const unlisted = [...metering.keys()].filter((name) => !listed.has(name)).sort() + return { facts, unlisted } +} + +/** The price of a credit: the account's rate when it can be read in dollars, otherwise list price. */ +export function creditPrice(rate: Row | undefined): { per_credit: number; source: "rate_sheet" | "list_price"; as_of?: string } { + const value = num(rate?.effective_rate) + if (value > 0 && String(rate?.currency ?? "").toUpperCase() === "USD") { + return { per_credit: value, source: "rate_sheet", as_of: String(rate?.rate_date ?? "") } + } + return { per_credit: LIST_PRICE_PER_CREDIT, source: "list_price" } +} + +/** `rules` defaults to the installed core's; tests pass their own. */ +export async function snowflakeSavings( + params: SnowflakeSavingsParams, + rules: RulesEngine | null = rulesEngine(), + context?: QueryContext, +): Promise { + const window = savingsWindow(params.now_ms ?? Date.now()) + const failed = (error: string): SnowflakeSavingsResult => ({ success: false, window, error }) + + const resolved = resolveFinopsWarehouse({ + requested: params.warehouse?.trim() || undefined, + supportedTypes: SUPPORTED_TYPES, + operationName: "The Snowflake savings check", + }) + if (resolved.kind === "error") return failed(resolved.error) + + if (!rules) { + return failed("This altimate-code build's altimate-core has no Snowflake savings rules. Update altimate-code, then try again.") + } + + // Stopping the session cancels the running query and starts no other. The check stops at its own + // limit, or earlier at the session's query cutoff (`altimate run --time-budget`). + const signal = context?.signal + const ownLimitAt = Date.now() + SAVINGS_RUN_LIMIT_MS + const sessionCutoff = context?.sessionID ? RunBudget.queryDeadline(context.sessionID) : undefined + const stopsAt = Math.min(ownLimitAt, sessionCutoff ?? Infinity) + const limitName = stopsAt < ownLimitAt ? "the run's time budget" : "the check's 9-minute limit" + + // Connecting can wait on the network or a sign-in: it counts toward the limit and yields to a stop. + const beforeStop = async (work: Promise, what: string): Promise => { + signal?.throwIfAborted() + let timer: ReturnType | undefined + let onAbort: (() => void) | undefined + try { + return await Promise.race([ + work, + new Promise((_, reject) => { + timer = setTimeout(() => reject(new Error(`${what} took longer than ${limitName}.`)), Math.max(0, stopsAt - Date.now())) + }), + new Promise((_, reject) => { + onAbort = () => reject(signal!.reason) + signal?.addEventListener("abort", onAbort, { once: true }) + }), + ]) + } finally { + clearTimeout(timer) + if (onAbort) signal?.removeEventListener("abort", onAbort) + work.catch(() => {}) + } + } + + try { + const connector = await beforeStop(Registry.get(resolved.warehouse), "Connecting to the warehouse") + const config = Registry.getConfig(resolved.warehouse) as Record | undefined + const limited: string[] = [] + const read = async (what: string, sql: string, rowLimit = ROW_LIMIT) => { + signal?.throwIfAborted() + const left = stopsAt - Date.now() + if (left <= 0) throw new Error(`The ${what} query was not started: it would run past ${limitName}.`) + const binds = Array.from({ length: sql.split("?").length - 1 }, () => window.end) + try { + const result = await connector.execute(sql, rowLimit, binds, { cancelAfterMs: left, signal }) + if (result.truncated) limited.push(what) + return records(result) + } catch (e) { + if (!isQueryTimeoutError(e)) throw e + const outcome = (e as { cancelled?: boolean }).cancelled + ? "and was cancelled on the warehouse" + : "and may still be running on the warehouse" + throw new Error(`The ${what} query ran past ${limitName} ${outcome}.`) + } + } + const warehouses = await read("warehouses", SNOWFLAKE_WAREHOUSES_SQL, SHOW_ROW_LIMIT) + if (warehouses.length === 0) return failed("SHOW WAREHOUSES lists no warehouse this connection's role can see.") + + // A view the role may not read stops only the checks that need it; the rest still report. + const skipped: { check: string; reason: string }[] = [] + let checksMissing = 0 + const reasonOf = (e: unknown) => (e instanceof Error ? e.message : String(e)).split("\n")[0]!.slice(0, 300) + const optional = async (check: string, what: string, sql: string) => { + try { + return await read(what, sql) + } catch (e) { + if (signal?.aborted) throw e + checksMissing++ + skipped.push({ check, reason: reasonOf(e) }) + return [] + } + } + // Most accounts cannot read the rate sheet (it needs organization access); list price stands in. + const rate = await read("credit rate", SNOWFLAKE_RATE_SQL).catch((e) => { + if (signal?.aborted) throw e + // An unreadable rate sheet is ordinary; a query that may still be running is worth naming. + if (reasonOf(e).includes("may still be running")) skipped.push({ check: "The account's credit rate", reason: reasonOf(e) }) + return [] as Row[] + }) + const metering = await read("metering", SNOWFLAKE_METERING_FACTS_SQL) + const periods = await optional( + "High Idle Time, Underutilized Cluster and the short-period signal of Downsize Warehouse", + "active periods", + SNOWFLAKE_PERIOD_FACTS_SQL, + ) + const restarts = await optional("Frequent Restart Overcharge", "restart events", SNOWFLAKE_RESTART_FACTS_SQL) + const queries = await optional("The short-query signal of Downsize Warehouse", "query history", SNOWFLAKE_QUERY_FACTS_SQL) + + // All four checks rest on these three queries: with none of them, nothing was assessed. + if (checksMissing === 3) return failed(`No check could run. ${skipped.at(-1)!.reason}`) + + const price = creditPrice(rate[0]) + const { facts, unlisted } = buildFacts({ warehouses, metering, periods, restarts, queries }) + if (unlisted.length > 0) { + skipped.push({ + check: `${unlisted.length} warehouse${unlisted.length === 1 ? "" : "s"} that billed credits`, + reason: `SHOW WAREHOUSES does not list ${unlisted.slice(0, 5).join(", ")}${unlisted.length > 5 ? ", …" : ""}: dropped, renamed, or not visible to the connection's role`, + }) + } + const report = rules(JSON.stringify({ lookback_days: LOOKBACK_DAYS, price_per_credit: price.per_credit, warehouses: facts })) + return { + success: true, + window, + warehouse: resolved.warehouse, + ran_on: config?.warehouse == null ? undefined : String(config.warehouse), + warehouses_checked: facts.length, + price, + skipped, + limited, + // What each warehouse is and costs now, so a later run can tell what changed. + warehouses: facts.map((one) => ({ + id: one.warehouse_id, + name: one.warehouse_name, + type: one.warehouse_type ?? "", + size: one.warehouse_size, + window_cost: one.usage_days.reduce((sum, day) => sum + (day.credits_used ?? 0), 0) * price.per_credit, + })), + report, + } + } catch (e) { + return failed(e instanceof Error ? e.message : String(e)) + } +} diff --git a/packages/opencode/src/altimate/native/types.ts b/packages/opencode/src/altimate/native/types.ts index c221a4cba9..ea9e52bfe6 100644 --- a/packages/opencode/src/altimate/native/types.ts +++ b/packages/opencode/src/altimate/native/types.ts @@ -699,6 +699,64 @@ export interface DatabricksSavingsResult { error?: string } +// --- FinOps: Snowflake warehouse savings --- + +export interface SnowflakeSavingsParams { + /** Snowflake connection name. If omitted, the first Snowflake connection is used. */ + warehouse?: string + /** Clock override for tests; the window ends on the UTC date six hours before this instant. */ + now_ms?: number +} + +/** One saving on one warehouse, as altimate-core reports it. Monthly figures are 0 when it has no dollar estimate. */ +export interface SnowflakeSavingsFinding { + check_id: "high_idle_time" | "downsize_warehouse" | "underutilized_cluster" | "frequent_restart_overcharge" + title: string + resource_type: "sql_warehouse" + resource_id: string + resource_name: string + workspace_id?: null + severity: "critical" | "high" | "medium" | "low" + daily_min: number + daily_max: number + monthly_min: number + monthly_max: number + evidence: Record & { + kind: "high_idle_time" | "downsize_warehouse" | "underutilized_cluster" | "frequent_restart" + } +} + +export interface SnowflakeSavingsReport { + lookback_days: number + price_per_credit: number + /** Largest monthly saving first; findings with no dollar estimate last. */ + findings: SnowflakeSavingsFinding[] + /** Warehouses with a dollar saving, each counted once at its largest low and high end. */ + total: { monthly_min: number; monthly_max: number; resources: number } + /** Always empty: no Snowflake rule prices alternatives. */ + what_if: DatabricksWhatIf[] +} + +export interface SnowflakeSavingsResult { + success: boolean + /** The days the facts cover: `start` inclusive, `end` exclusive (YYYY-MM-DD, UTC). */ + window: { start: string; end: string } + /** The connection that ran the check, and the warehouse its configuration names. */ + warehouse?: string + ran_on?: string + warehouses_checked?: number + /** The dollars one credit is priced at, and where that came from. */ + price?: { per_credit: number; source: "rate_sheet" | "list_price"; as_of?: string } + /** Checks whose views the connection could not read, and billed warehouses it could not list. */ + skipped?: { check: string; reason: string }[] + /** What a query read only the first rows of, e.g. `active periods`. */ + limited?: string[] + /** Each warehouse as it is now, and its cost over the window. */ + warehouses?: { id: string; workspace_id?: string | null; name: string; type: string; size: string | null; window_cost: number }[] + report?: SnowflakeSavingsReport + error?: string +} + // --- FinOps: Warehouse Advisor --- export interface WarehouseAdvisorParams { @@ -1319,6 +1377,7 @@ export const BridgeMethods = { "finops.expensive_queries": {} as { params: ExpensiveQueriesParams; result: ExpensiveQueriesResult }, "finops.warehouse_advice": {} as { params: WarehouseAdvisorParams; result: WarehouseAdvisorResult }, "finops.databricks_savings": {} as { params: DatabricksSavingsParams; result: DatabricksSavingsResult }, + "finops.snowflake_savings": {} as { params: SnowflakeSavingsParams; result: SnowflakeSavingsResult }, "finops.unused_resources": {} as { params: UnusedResourcesParams; result: UnusedResourcesResult }, "finops.role_grants": {} as { params: RoleGrantsParams; result: RoleGrantsResult }, "finops.role_hierarchy": {} as { params: RoleHierarchyParams; result: RoleHierarchyResult }, diff --git a/packages/opencode/src/altimate/tool-cli/allow-list.ts b/packages/opencode/src/altimate/tool-cli/allow-list.ts index f62a6b2d08..8e0e6b56fb 100644 --- a/packages/opencode/src/altimate/tool-cli/allow-list.ts +++ b/packages/opencode/src/altimate/tool-cli/allow-list.ts @@ -50,6 +50,7 @@ import { DbtUnitTestGenTool } from "../tools/dbt-unit-test-gen" import { DocsShareCommentsTool } from "../tools/docs-share-comments" import { FinopsAnalyzeCreditsTool } from "../tools/finops-analyze-credits" import { FinopsDatabricksSavingsTool } from "../tools/finops-databricks-savings" +import { FinopsSnowflakeSavingsTool } from "../tools/finops-snowflake-savings" import { FinopsExpensiveQueriesTool } from "../tools/finops-expensive-queries" import { FinopsQueryHistoryTool } from "../tools/finops-query-history" import { FinopsRoleGrantsTool, FinopsRoleHierarchyTool, FinopsUserRolesTool } from "../tools/finops-role-access" @@ -131,6 +132,7 @@ export const EXPOSED: Record = { docs_share_comments: DocsShareCommentsTool, finops_analyze_credits: FinopsAnalyzeCreditsTool, finops_databricks_savings: FinopsDatabricksSavingsTool, + finops_snowflake_savings: FinopsSnowflakeSavingsTool, finops_expensive_queries: FinopsExpensiveQueriesTool, finops_query_history: FinopsQueryHistoryTool, finops_role_grants: FinopsRoleGrantsTool, diff --git a/packages/opencode/src/altimate/tools/finops-databricks-savings.ts b/packages/opencode/src/altimate/tools/finops-databricks-savings.ts index 4b8cdf539d..fa1e10ca06 100644 --- a/packages/opencode/src/altimate/tools/finops-databricks-savings.ts +++ b/packages/opencode/src/altimate/tools/finops-databricks-savings.ts @@ -22,13 +22,13 @@ type SavingsMetadata = { needs_confirmation?: boolean } -function dollars(value: number): string { +export function dollars(value: number): string { return value >= 100 ? `$${Math.round(value).toLocaleString("en-US")}` : `$${value.toFixed(2)}` } -function range(low: number, high: number): string { +export function range(low: number, high: number): string { return dollars(low) === dollars(high) ? dollars(high) : `${dollars(low)}–${dollars(high)}` } diff --git a/packages/opencode/src/altimate/tools/finops-snowflake-savings.ts b/packages/opencode/src/altimate/tools/finops-snowflake-savings.ts new file mode 100644 index 0000000000..339c87f3fa --- /dev/null +++ b/packages/opencode/src/altimate/tools/finops-snowflake-savings.ts @@ -0,0 +1,172 @@ +// altimate_change - new file +import z from "zod" +import { Tool } from "../../tool/tool" +import { Dispatcher } from "../native" +import type { SnowflakeSavingsFinding, SnowflakeSavingsReport, SnowflakeSavingsResult } from "../native/types" +import { dollars, range } from "./finops-databricks-savings" + +/** One shape for every outcome, so a reader of the metadata never guesses which fields exist. */ +type SavingsMetadata = { + success: boolean + window?: { start: string; end: string } + warehouse?: string + ran_on?: string + warehouses_checked?: number + price?: SnowflakeSavingsResult["price"] + skipped?: { check: string; reason: string }[] + limited?: string[] + warehouses?: SnowflakeSavingsResult["warehouses"] + report?: SnowflakeSavingsReport + error?: string + /** True when the check did not run because `confirm` was not set. */ + needs_confirmation?: boolean +} + +const count = (value: unknown) => Number(value).toLocaleString("en-US") +const days = (value: unknown) => { + const n = Array.isArray(value) ? value.length : 0 + return `${n} day${n === 1 ? "" : "s"}` +} +const hours = (seconds: unknown) => (Number(seconds) / 3_600).toFixed(1) + +/** What a finding rests on, one short line each. */ +export function evidenceLines(finding: SnowflakeSavingsFinding): string[] { + const e = finding.evidence as Record & { kind: string } + switch (e.kind) { + case "high_idle_time": + return [ + `${Math.round(Number(e.idle_ratio) * 100)}% idle on ${days(e.flagged_days)} above 40%: ${hours(e.idle_seconds)} of ${hours(e.active_seconds)} active hours`, + `${Number(e.credits_used).toFixed(2)} credits on those days, at $${Number(e.price_per_credit).toFixed(2)} a credit`, + ] + case "downsize_warehouse": { + const signals = [ + e.short_period_percent == null ? null : `${Number(e.short_period_percent).toFixed(1)}% of its active periods lasted 80 seconds or less`, + e.sub_five_second_percent == null + ? null + : `${Number(e.sub_five_second_percent).toFixed(1)}% of ${count(e.query_count)} queries ran in 5 seconds or less`, + ].filter(Boolean) + return [ + `${e.current_size} could be ${e.recommended_size}: ${signals.join("; ")}`, + `The saving is a share of $${Number(e.cost_basis).toFixed(2)} over the window`, + ] + } + case "underutilized_cluster": + return [ + `Cluster ${e.cluster_number} of ${e.max_cluster_count} was barely needed on ${days(e.flagged_days)} (up to ${Number(e.highest_idle_percent).toFixed(0)}% idle)`, + `Lowering the maximum to ${e.recommended_max_cluster_count} cluster${e.recommended_max_cluster_count === 1 ? "" : "s"} may add some queueing; no dollar estimate`, + ] + case "frequent_restart": + return [ + `${count(e.bounces)} suspend-and-resume bounces inside a billed minute on ${days(e.flagged_days)}`, + `${count(e.double_billed_seconds)} seconds billed twice at ${e.credits_per_hour} credits an hour`, + ] + default: + return [] + } +} + +function priceLine(result: SnowflakeSavingsResult): string { + const price = result.price + if (!price) return "" + return price.source === "rate_sheet" + ? `$${price.per_credit.toFixed(2)} a credit from the account's rate sheet` + : `$${price.per_credit.toFixed(2)} a credit, list price (no dollar rate could be read from the account's rate sheet)` +} + +function headline(report: SnowflakeSavingsReport): string { + const { monthly_min, monthly_max, resources } = report.total + if (resources > 0) return `${range(monthly_min, monthly_max)} a month on ${resources} warehouse${resources === 1 ? "" : "s"}` + if (report.findings.length === 0) return "no savings found" + return `${report.findings.length} finding${report.findings.length === 1 ? "" : "s"}, no dollar estimate` +} + +export function formatSavings(result: SnowflakeSavingsResult): string { + const report = result.report + if (!report) return "No report." + const checked = result.warehouses_checked ?? 0 + const lines: string[] = [ + `Snowflake savings: ${checked} warehouse${checked === 1 ? "" : "s"} checked, the ${report.lookback_days} days to ${result.window.end}, at ${priceLine(result)}.`, + ] + for (const one of result.skipped ?? []) lines.push(`Not checked: ${one.check} (${one.reason})`) + for (const what of result.limited ?? []) lines.push(`Only part of the ${what} could be read; the rest was not checked.`) + if (report.findings.length === 0) { + lines.push(result.skipped?.length || result.limited?.length ? "No savings found in what could be checked." : "No savings found.") + return lines.join("\n") + } + if (report.total.resources > 0) lines.push(`Total: ${headline(report)}, each counted once at its largest saving.`) + lines.push("") + for (const finding of report.findings) { + const amount = finding.monthly_max > 0 ? `${range(finding.monthly_min, finding.monthly_max)} a month` : "no dollar estimate" + lines.push(`${finding.resource_name} (warehouse) · ${finding.title} · ${amount} (${finding.severity})`) + for (const line of evidenceLines(finding)) lines.push(` ${line}`) + } + return lines.join("\n") +} + +export const FinopsSnowflakeSavingsTool = Tool.define("finops_snowflake_savings", { + description: + "Check a Snowflake account for savings with Altimate's rules: warehouses that sit idle, could run one size smaller, keep an unneeded extra cluster, or restart so often they are billed twice. Read-only: SHOW WAREHOUSES plus five queries over seven days of SNOWFLAKE.ACCOUNT_USAGE and the rate sheet, on the connection's warehouse, which Snowflake resumes if it is suspended and auto-resume is on. Returns each finding with its monthly dollar range and evidence. Ask the user before running it, and pass confirm: true once they agree; without it nothing runs.", + parameters: z.object({ + warehouse: z + .string() + .optional() + .describe("Snowflake connection name. Optional — omit to use the first configured Snowflake connection."), + confirm: z + .boolean() + .optional() + .describe( + "Set to true only after the user has agreed to run the check. It runs six queries on their Snowflake account; five use the connection's warehouse, which Snowflake resumes if it is suspended and auto-resume is on, and which bills while it runs; the check stops after 9 minutes, cancelling any query still running.", + ), + }), + async execute(args, ctx) { + if (args.confirm !== true) { + // Asking first is the expected first step, not a failed run. + return { + title: "Snowflake Savings: not run, needs the user's go-ahead", + metadata: { success: true, needs_confirmation: true } as SavingsMetadata, + output: + "Not run. The Snowflake savings check runs six queries over seven days of account usage; five use the connection's warehouse, " + + "which Snowflake resumes if it is suspended and auto-resume is on, and which bills while it runs. It stops after 9 minutes, cancelling any query still running. " + + "Ask the user whether to run it; if they agree, call this tool again with confirm: true.", + } + } + try { + const result = await Dispatcher.call( + "finops.snowflake_savings", + { warehouse: args.warehouse }, + { signal: ctx.abort, sessionID: ctx.sessionID }, + ) + if (!result.success || !result.report) { + const error = result.error ?? "Unknown error" + return { + title: "Snowflake Savings: FAILED", + metadata: { success: false, window: result.window, error } as SavingsMetadata, + output: `The Snowflake savings check failed: ${error}`, + } + } + return { + title: `Snowflake Savings: ${headline(result.report)}`, + metadata: { + success: true, + window: result.window, + warehouse: result.warehouse, + ran_on: result.ran_on, + warehouses_checked: result.warehouses_checked, + price: result.price, + skipped: result.skipped, + limited: result.limited, + warehouses: result.warehouses, + report: result.report, + } as SavingsMetadata, + output: formatSavings(result), + } + } catch (e) { + const msg = e instanceof Error ? e.message : String(e) + return { + title: "Snowflake Savings: ERROR", + metadata: { success: false, error: msg } as SavingsMetadata, + output: `The Snowflake savings check failed: ${msg}`, + } + } + }, +}) diff --git a/packages/opencode/src/tool/registry.ts b/packages/opencode/src/tool/registry.ts index db2e3b238e..40842ec265 100644 --- a/packages/opencode/src/tool/registry.ts +++ b/packages/opencode/src/tool/registry.ts @@ -107,6 +107,7 @@ import { FinopsAnalyzeCreditsTool } from "../altimate/tools/finops-analyze-credi import { FinopsExpensiveQueriesTool } from "../altimate/tools/finops-expensive-queries" import { FinopsWarehouseAdviceTool } from "../altimate/tools/finops-warehouse-advice" import { FinopsDatabricksSavingsTool } from "../altimate/tools/finops-databricks-savings" +import { FinopsSnowflakeSavingsTool } from "../altimate/tools/finops-snowflake-savings" import { FinopsUnusedResourcesTool } from "../altimate/tools/finops-unused-resources" import { FinopsRoleGrantsTool, @@ -492,6 +493,7 @@ export namespace ToolRegistry { FinopsExpensiveQueriesTool, FinopsWarehouseAdviceTool, FinopsDatabricksSavingsTool, + FinopsSnowflakeSavingsTool, FinopsUnusedResourcesTool, FinopsRoleGrantsTool, FinopsRoleHierarchyTool, diff --git a/packages/opencode/test/agent/analyst-data-explorer.test.ts b/packages/opencode/test/agent/analyst-data-explorer.test.ts index 8943948662..4d4a43f56c 100644 --- a/packages/opencode/test/agent/analyst-data-explorer.test.ts +++ b/packages/opencode/test/agent/analyst-data-explorer.test.ts @@ -424,6 +424,7 @@ const analystConfigEnabled = new Set([ "finops_query_history", "finops_role_grants", "finops_role_hierarchy", + "finops_snowflake_savings", "finops_unused_resources", "finops_user_roles", "finops_warehouse_advice", diff --git a/packages/opencode/test/altimate/assessment-zero-egress.test.ts b/packages/opencode/test/altimate/assessment-zero-egress.test.ts index ff9e0ac83f..92a28edbea 100644 --- a/packages/opencode/test/altimate/assessment-zero-egress.test.ts +++ b/packages/opencode/test/altimate/assessment-zero-egress.test.ts @@ -4,7 +4,8 @@ import { afterAll, beforeAll, describe, expect, test } from "bun:test" import http from "node:http" import type { AddressInfo } from "node:net" -import { mkdir, mkdtemp, readFile, readdir, rm, stat, writeFile } from "node:fs/promises" +import { createRequire } from "node:module" +import { mkdir, mkdtemp, readFile, readdir, realpath, rm, stat, symlink, writeFile } from "node:fs/promises" import { tmpdir } from "node:os" import path from "node:path" @@ -137,6 +138,56 @@ describe("assessment mode on the packaged CLI", () => { expect(JSON.parse(result.stdout).output).toContain("call this tool again with confirm: true") }, 120_000) + posixOnly("the Snowflake check: unconfirmed, with no warehouse, and with an unreachable one, no other host is contacted", async () => { + const snowflake = (args: string) => ["tool", "run", "finops_snowflake_savings", "--args", args, "--json", "--no-daemon"] + const unconfirmed = await run(snowflake("{}"), assessment) + expect(unconfirmed.hosts).toEqual([]) + expect(JSON.parse(unconfirmed.stdout).output).toContain("call this tool again with confirm: true") + + const noWarehouse = await run(snowflake('{"confirm":true}'), assessment) + expect(noWarehouse.hosts).toEqual([]) + expect(JSON.parse(noWarehouse.stdout).status).toBe("error") + + const account = "egress-test.invalid" + const unreachable = await run(snowflake('{"confirm":true}'), assessment, async (_home, project) => { + await mkdir(path.join(project, ".altimate-code"), { recursive: true }) + await writeFile( + path.join(project, ".altimate-code", "connections.json"), + JSON.stringify({ sf: { type: "snowflake", account, user: "u", password: "unused", warehouse: "W" } }), + ) + }) + expect(unreachable.hosts.filter((host) => !host.includes(account))).toEqual([]) + expect(JSON.parse(unreachable.stdout).output).not.toContain("none are set up") + }, 360_000) + + posixOnly("with the Snowflake driver installed, assessment mode keeps it off the cloud metadata services", async () => { + const account = "egress-test" + const args = ["tool", "run", "finops_snowflake_savings", "--args", '{"confirm":true}', "--json", "--no-daemon"] + // The driver the packaged CLI would install, linked in with its dependencies. + const sdk = createRequire(path.join(root, "../drivers/package.json")).resolve("snowflake-sdk/package.json") + const dependencies = path.dirname(path.dirname(await realpath(sdk))) + const setup = async (home: string, project: string) => { + const drivers = path.join(home, ".local/share/altimate-code/drivers") + await mkdir(drivers, { recursive: true }) + await writeFile(path.join(drivers, "package.json"), JSON.stringify({ name: "drivers", private: true, dependencies: { "snowflake-sdk": "*" } })) + await symlink(dependencies, path.join(drivers, "node_modules")) + await mkdir(path.join(project, ".altimate-code"), { recursive: true }) + await writeFile( + path.join(project, ".altimate-code", "connections.json"), + JSON.stringify({ sf: { type: "snowflake", account, user: "u", password: "unused", warehouse: "W" } }), + ) + } + const metadata = (hosts: string[]) => hosts.filter((host) => /169\.254\.169\.254|metadata\.google\.internal|sts\.|login\.microsoftonline/.test(host)) + + // Control: without the mode the SDK does probe, so the proxy would see it. + const control = await run(args, { ALTIMATE_TELEMETRY_DISABLED: "true" }, setup) + expect(metadata(control.hosts).length).toBeGreaterThan(0) + + const result = await run(args, assessment, setup) + expect(result.hosts.some((host) => host.includes(account))).toBe(true) + expect(result.hosts.filter((host) => !host.includes(account))).toEqual([]) + }, 360_000) + posixOnly("startup path (`tool list`): no host is contacted", async () => { const result = await run(["tool", "list"], assessment) expect(result.code).toBe(0) diff --git a/packages/opencode/test/altimate/finops-snowflake-savings.test.ts b/packages/opencode/test/altimate/finops-snowflake-savings.test.ts new file mode 100644 index 0000000000..009dca6e80 --- /dev/null +++ b/packages/opencode/test/altimate/finops-snowflake-savings.test.ts @@ -0,0 +1,707 @@ +import { afterEach, beforeEach, describe, expect, spyOn, test } from "bun:test" +import { initTool } from "./tool-fixture" +import * as Dispatcher from "../../src/altimate/native/dispatcher" +import * as Registry from "../../src/altimate/native/connections/registry" +import { + LIST_PRICE_PER_CREDIT, + ROW_LIMIT, + SAVINGS_RUN_LIMIT_MS, + SHOW_ROW_LIMIT, + SNOWFLAKE_METERING_FACTS_SQL, + SNOWFLAKE_PERIOD_FACTS_SQL, + SNOWFLAKE_QUERY_FACTS_SQL, + SNOWFLAKE_RATE_SQL, + SNOWFLAKE_RESTART_FACTS_SQL, + SNOWFLAKE_WAREHOUSES_SQL, + buildFacts, + creditPrice, + rulesEngine, + snowflakeSavings, +} from "../../src/altimate/native/finops/snowflake-savings" +import { FinopsSnowflakeSavingsTool, formatSavings } from "../../src/altimate/tools/finops-snowflake-savings" +import type { SnowflakeSavingsResult } from "../../src/altimate/native/types" +import { RunBudget } from "../../src/altimate/run-budget" + +function stubCtx(): any { + return { sessionID: "test", messageID: "test", agent: "test", abort: new AbortController().signal, metadata: () => {}, ask: async () => {} } +} + +const finding = (check_id: string, evidence: Record, monthly_min: number, monthly_max: number) => ({ + check_id, + title: { high_idle_time: "High Idle Time", downsize_warehouse: "Downsize Warehouse", underutilized_cluster: "Underutilized Cluster", frequent_restart_overcharge: "Frequent Restart Overcharge" }[check_id], + resource_type: "sql_warehouse", + resource_id: "7", + resource_name: "BI_WH", + workspace_id: null, + severity: "medium", + daily_min: monthly_min / 30, + daily_max: monthly_max / 30, + monthly_min, + monthly_max, + evidence, +}) + +const REPORT = { + success: true, + window: { start: "2026-11-11", end: "2026-11-18" }, + warehouse: "sf", + ran_on: "BI_WH", + warehouses_checked: 3, + price: { per_credit: 3, source: "rate_sheet", as_of: "2026-11-17" }, + skipped: [], + limited: [], + warehouses: [{ id: "7", name: "BI_WH", type: "STANDARD", size: "Large", window_cost: 42 }], + report: { + lookback_days: 7, + price_per_credit: 3, + findings: [ + finding("downsize_warehouse", { kind: "downsize_warehouse", current_size: "Large", recommended_size: "Medium", short_period_percent: 95, sub_five_second_percent: 91.5, query_count: 1200, cost_basis: 42, price_per_credit: 3 }, 54, 90), + finding("high_idle_time", { kind: "high_idle_time", idle_ratio: 0.6, idle_seconds: 7200, active_seconds: 12000, flagged_days: ["2026-11-11", "2026-11-12"], credits_used: 10, price_per_credit: 3, wasted_cost: 18 }, 54, 77.14), + finding("frequent_restart_overcharge", { kind: "frequent_restart", bounces: 12, double_billed_seconds: 300, credits_per_hour: 4, flagged_days: ["2026-11-11"], price_per_credit: 3 }, 4.29, 6.43), + finding("underutilized_cluster", { kind: "underutilized_cluster", cluster_number: 2, max_cluster_count: 2, recommended_max_cluster_count: 1, highest_idle_percent: 70, flagged_days: ["2026-11-13"] }, 0, 0), + ], + total: { monthly_min: 54, monthly_max: 90, resources: 1 }, + what_if: [], + }, +} as unknown as SnowflakeSavingsResult + +describe("finops_snowflake_savings tool", () => { + beforeEach(() => Dispatcher.reset()) + + test("gives Claude the total, each finding with its evidence, and the whole report as metadata", async () => { + Dispatcher.register("finops.snowflake_savings" as any, async () => REPORT) + const tool = await initTool(FinopsSnowflakeSavingsTool) + const result = await tool.execute({ warehouse: "sf", confirm: true }, stubCtx()) + expect(result.title).toBe("Snowflake Savings: $54.00–$90.00 a month on 1 warehouse") + expect(result.metadata.report).toEqual(REPORT.report) + expect(result.metadata.price).toEqual(REPORT.price) + expect(result.output).toContain("3 warehouses checked, the 7 days to 2026-11-18, at $3.00 a credit from the account's rate sheet") + expect(result.output).toContain("BI_WH (warehouse) · Downsize Warehouse · $54.00–$90.00 a month (medium)") + expect(result.output).toContain("Large could be Medium: 95.0% of its active periods lasted 80 seconds or less; 91.5% of 1,200 queries ran in 5 seconds or less") + expect(result.output).toContain("60% idle on 2 days above 40%: 2.0 of 3.3 active hours") + expect(result.output).toContain("12 suspend-and-resume bounces inside a billed minute on 1 day") + expect(result.output).toContain("Underutilized Cluster · no dollar estimate") + expect(result.output).toContain("Lowering the maximum to 1 cluster may add some queueing") + }) + + test("without the user's go-ahead nothing runs: the agent is told to ask first", async () => { + let ran = false + Dispatcher.register("finops.snowflake_savings" as any, async () => { + ran = true + return REPORT + }) + const tool = await initTool(FinopsSnowflakeSavingsTool) + for (const args of [{}, { confirm: false }, { warehouse: "sf" }]) { + const result = await tool.execute(args, stubCtx()) + expect(result.title).toBe("Snowflake Savings: not run, needs the user's go-ahead") + expect(result.metadata).toMatchObject({ success: true, needs_confirmation: true }) + expect(result.metadata.report).toBeUndefined() + expect(result.output).toContain("Snowflake resumes if it is suspended") + expect(result.output).toContain("It stops after 9 minutes, cancelling any query still running.") + expect(result.output).toContain("Ask the user whether to run it; if they agree, call this tool again with confirm: true.") + } + expect(ran).toBe(false) + expect(tool.description).toEndWith("Ask the user before running it, and pass confirm: true once they agree; without it nothing runs.") + }) + + test("the session's stop signal reaches the check", async () => { + let seen: unknown + Dispatcher.register("finops.snowflake_savings" as any, async (_params: unknown, context: unknown) => { + seen = context + return REPORT + }) + const tool = await initTool(FinopsSnowflakeSavingsTool) + const ctx = stubCtx() + await tool.execute({ confirm: true }, ctx) + expect(seen).toEqual({ signal: ctx.abort, sessionID: "test" }) + }) + + test("findings with no dollar figure are not presented as a saving", async () => { + const only = REPORT.report!.findings.slice(3) + Dispatcher.register("finops.snowflake_savings" as any, async () => ({ + ...REPORT, + report: { ...REPORT.report, findings: only, total: { monthly_min: 0, monthly_max: 0, resources: 0 } }, + })) + const tool = await initTool(FinopsSnowflakeSavingsTool) + const result = await tool.execute({ confirm: true }, stubCtx()) + expect(result.title).toBe("Snowflake Savings: 1 finding, no dollar estimate") + expect(result.output).not.toContain("Total:") + }) + + test("no findings says so; list price and skipped checks are stated", () => { + const output = formatSavings({ + ...REPORT, + price: { per_credit: 3, source: "list_price" }, + skipped: [{ check: "Frequent Restart Overcharge", reason: "not authorized" }], + limited: ["active periods"], + report: { ...REPORT.report!, findings: [], total: { monthly_min: 0, monthly_max: 0, resources: 0 } }, + }) + expect(output).toContain("$3.00 a credit, list price (no dollar rate could be read from the account's rate sheet)") + expect(output).toContain("Not checked: Frequent Restart Overcharge (not authorized)") + expect(output).toContain("Only part of the active periods could be read; the rest was not checked.") + // Some checks did not run, so this is not a clean bill of health. + expect(output).toContain("No savings found in what could be checked.") + expect(formatSavings({ ...REPORT, warehouses_checked: 1, report: { ...REPORT.report!, findings: [] } })).toContain("1 warehouse checked") + expect(formatSavings({ ...REPORT, report: { ...REPORT.report!, findings: [] } })).toEndWith("No savings found.") + }) + + test("a run with nothing to report says so in its title, and a partial read qualifies the result", async () => { + const empty = { ...REPORT.report!, findings: [], total: { monthly_min: 0, monthly_max: 0, resources: 0 } } + Dispatcher.register("finops.snowflake_savings" as any, async () => ({ ...REPORT, report: empty })) + const tool = await initTool(FinopsSnowflakeSavingsTool) + expect((await tool.execute({ confirm: true }, stubCtx())).title).toBe("Snowflake Savings: no savings found") + expect(formatSavings({ ...REPORT, limited: ["metering"], report: empty })).toEndWith("No savings found in what could be checked.") + }) + + test("a failure is reported with its reason, not as no savings", async () => { + Dispatcher.register("finops.snowflake_savings" as any, async () => ({ + success: false, + window: REPORT.window, + error: "SQL access control error: Insufficient privileges", + })) + const tool = await initTool(FinopsSnowflakeSavingsTool) + const result = await tool.execute({ confirm: true }, stubCtx()) + expect(result.title).toBe("Snowflake Savings: FAILED") + expect(result.metadata.error).toContain("Insufficient privileges") + expect(result.output).not.toContain("No savings found") + }) +}) + +describe("the fact queries", () => { + const all = [SNOWFLAKE_METERING_FACTS_SQL, SNOWFLAKE_PERIOD_FACTS_SQL, SNOWFLAKE_RESTART_FACTS_SQL, SNOWFLAKE_QUERY_FACTS_SQL, SNOWFLAKE_RATE_SQL] + + test("each is one read-only statement whose only parameter is the window end", () => { + for (const sql of all) { + expect(sql.trim()).toMatch(/^(WITH|SELECT)\b/) + expect(sql).not.toMatch(/;\s*\S/) + expect(sql).not.toMatch(/\b(INSERT|UPDATE|DELETE|MERGE|CREATE|ALTER|DROP|CALL|GRANT)\b/) + expect(sql).toContain("?") + } + expect(SNOWFLAKE_WAREHOUSES_SQL).toBe("SHOW WAREHOUSES") + }) + + test("every timestamp is read as UTC, so a session time zone cannot move a day", () => { + for (const sql of all.slice(0, 4)) { + expect(sql).toContain("CONVERT_TIMEZONE('UTC'") + expect(sql).toContain("+0000") + } + }) + + test("dates are parsed and printed in a fixed format, whatever the session's settings", () => { + for (const sql of all) expect(sql).not.toMatch(/TO_TIMESTAMP_NTZ\(\?\)/) + for (const sql of all) { + for (const call of sql.match(/TO_VARCHAR\([^\n]*/g) ?? []) expect(call).toContain("'YYYY-MM-DD'") + } + }) + + test("the measured thresholds are the hosted rules'", () => { + expect(SNOWFLAKE_PERIOD_FACTS_SQL).toContain("COUNT_IF(active_seconds <= 80)") + expect(SNOWFLAKE_PERIOD_FACTS_SQL).toContain("DATEDIFF('millisecond', valid_from, valid_to) >= 1000") + expect(SNOWFLAKE_PERIOD_FACTS_SQL).toContain("DATEDIFF('second', valid_from, valid_to)") + expect(SNOWFLAKE_QUERY_FACTS_SQL).toContain("COUNT_IF(q.execution_ms <= 5000)") + expect(SNOWFLAKE_QUERY_FACTS_SQL).toContain("SUM(compute_credits) * 0.1") + }) + + test("the windows and filters are the ones the rules are defined on", () => { + // Seven days plus the hour the first idle window starts early; system warehouse 0 is left out. + expect(SNOWFLAKE_METERING_FACTS_SQL).toContain("DATEADD(hour, -169,") + expect(SNOWFLAKE_METERING_FACTS_SQL).toContain("WAREHOUSE_ID > 0") + expect(SNOWFLAKE_METERING_FACTS_SQL).toContain("TO_VARCHAR(DATEADD(hour, 1, hour_start)::DATE, 'YYYY-MM-DD'), NULL") + expect(SNOWFLAKE_PERIOD_FACTS_SQL).toContain("TO_VARCHAR(DATEADD(hour, 1, valid_from)::DATE, 'YYYY-MM-DD'), NULL") + expect(SNOWFLAKE_PERIOD_FACTS_SQL).toContain("EVENT_NAME IN ('RESUME_CLUSTER', 'SUSPEND_CLUSTER') AND EVENT_STATE = 'STARTED'") + expect(SNOWFLAKE_PERIOD_FACTS_SQL).toContain("q.cluster_number = p.cluster_number") + expect(SNOWFLAKE_QUERY_FACTS_SQL).toContain("q.et >= DATEADD(day, -7,") + expect(SNOWFLAKE_RATE_SQL).toContain("ACCOUNT_NAME = CURRENT_ACCOUNT_NAME()") + expect(SNOWFLAKE_RATE_SQL).toContain("SERVICE_TYPE = 'WAREHOUSE_METERING' AND USAGE_TYPE = 'compute'") + }) + + test("a bounce is the full resume, suspend, resume sequence on distinct events", () => { + expect(SNOWFLAKE_RESTART_FACTS_SQL).toContain("SELECT DISTINCT") + expect(SNOWFLAKE_RESTART_FACTS_SQL).toContain( + "event_name = 'RESUME_WAREHOUSE' AND n1 = 'WAREHOUSE_CONSISTENT' AND n2 = 'SUSPEND_WAREHOUSE'", + ) + expect(SNOWFLAKE_RESTART_FACTS_SQL).toContain("n3 = 'WAREHOUSE_CONSISTENT' AND n4 = 'RESUME_WAREHOUSE'") + expect(SNOWFLAKE_RESTART_FACTS_SQL).toContain("DATEDIFF('second', ts, t4) < 60") + }) +}) + +describe("building the facts", () => { + const show = (over: Record = {}) => ({ name: "BI_WH", size: "Large", type: "STANDARD", min_cluster_count: 1, max_cluster_count: "2", ...over }) + + test("rows become one entry per listed warehouse, with database values coerced", () => { + const { facts, unlisted } = buildFacts({ + warehouses: [show(), show({ name: "IDLE_WH", size: "X-Small", max_cluster_count: 1 })], + metering: [ + { warehouse_id: 7, warehouse_name: "BI_WH", usage_day: "2026-11-11", idle_day: "2026-11-11", credits_used: "1.5" }, + { warehouse_id: 7, warehouse_name: "BI_WH", usage_day: "2026-11-11", idle_day: "2026-11-12", credits_used: 0.5 }, + { warehouse_id: 7, warehouse_name: "BI_WH", usage_day: null, idle_day: "2026-11-11", credits_used: 2 }, + ], + periods: [ + { warehouse_id: 7, warehouse_name: "BI_WH", cluster_number: 1, usage_day: "2026-11-11", idle_day: "2026-11-11", periods: 3, short_periods: "2", active_seconds: "100", idle_seconds: 40 }, + { warehouse_id: 7, warehouse_name: "BI_WH", cluster_number: 2, usage_day: "2026-11-11", idle_day: "2026-11-11", periods: 1, short_periods: 1, active_seconds: 50, idle_seconds: 45 }, + ], + restarts: [{ warehouse_id: 7, warehouse_name: "BI_WH", day: "2026-11-11", gap_seconds: 30, bounces: 4n }], + queries: [{ warehouse_id: 7, warehouse_name: "BI_WH", query_count: "10", sub_five_second_queries: 9, sub_five_second_credits: "0.25" }], + }) + expect(unlisted).toEqual([]) + expect(facts[0]).toEqual({ + warehouse_id: "7", + warehouse_name: "BI_WH", + warehouse_size: "Large", + warehouse_type: "STANDARD", + min_cluster_count: 1, + max_cluster_count: 2, + query_count: 10, + sub_five_second_queries: 9, + sub_five_second_credits: 0.25, + usage_days: [{ day: "2026-11-11", credits_used: 2, active_periods: 4, short_periods: 3 }], + idle_days: [ + { day: "2026-11-11", active_seconds: 150, idle_seconds: 85, credits_used: 3.5 }, + { day: "2026-11-12", active_seconds: 0, idle_seconds: 0, credits_used: 0.5 }, + ], + cluster_days: [ + { + day: "2026-11-11", + clusters: [ + { cluster_number: 1, idle_seconds: 40, busy_seconds: 60 }, + { cluster_number: 2, idle_seconds: 45, busy_seconds: 5 }, + ], + }, + ], + restart_days: [{ day: "2026-11-11", gaps: [{ gap_seconds: 30, bounces: 4 }] }], + }) + // A listed warehouse with no usage is still an entry, under its name. + expect(facts[1]).toMatchObject({ warehouse_id: "IDLE_WH", usage_days: [], query_count: 0 }) + }) + + test("a billed day with no periods keeps its credits, and a day with periods but no billing has none", () => { + const { facts } = buildFacts({ + warehouses: [show()], + metering: [{ warehouse_id: 7, warehouse_name: "BI_WH", usage_day: "2026-11-11", idle_day: null, credits_used: 1 }], + periods: [{ warehouse_id: 7, warehouse_name: "BI_WH", cluster_number: 1, usage_day: "2026-11-12", idle_day: null, periods: 1, short_periods: 0, active_seconds: 9, idle_seconds: 1 }], + restarts: [], + queries: [], + }) + expect(facts[0]!.usage_days).toEqual([ + { day: "2026-11-11", credits_used: 1, active_periods: 0, short_periods: 0 }, + { day: "2026-11-12", credits_used: null, active_periods: 1, short_periods: 0 }, + ]) + }) + + test("a recreated warehouse keeps only its newest id's usage", () => { + const { facts } = buildFacts({ + warehouses: [show()], + metering: [ + { warehouse_id: 7, warehouse_name: "BI_WH", usage_day: "2026-11-11", idle_day: null, credits_used: 100 }, + { warehouse_id: 9, warehouse_name: "BI_WH", usage_day: "2026-11-12", idle_day: null, credits_used: 1 }, + ], + periods: [], + restarts: [], + queries: [], + }) + expect(facts[0]!.warehouse_id).toBe("9") + expect(facts[0]!.usage_days).toEqual([{ day: "2026-11-12", credits_used: 1, active_periods: 0, short_periods: 0 }]) + }) + + const meter = (id: number, name: string, day: string, credits = 1) => ({ warehouse_id: id, warehouse_name: name, usage_day: day, idle_day: null, credits_used: credits }) + const none = { periods: [], restarts: [], queries: [] } + + test("a renamed warehouse keeps its usage under its new name", () => { + const { facts, unlisted } = buildFacts({ + warehouses: [show({ name: "ETL_NEW" })], + metering: [meter(7, "ETL", "2026-11-11", 1), meter(7, "ETL_NEW", "2026-11-12", 2)], + ...none, + }) + expect(unlisted).toEqual([]) + expect(facts[0]).toMatchObject({ warehouse_id: "7", warehouse_name: "ETL_NEW" }) + expect(facts[0]!.usage_days.map((day) => day.credits_used)).toEqual([1, 2]) + }) + + test("a rename's query and restart rows are added, not read from one name or repeated", () => { + const { facts } = buildFacts({ + warehouses: [show({ name: "NEW" })], + metering: [meter(7, "AOLD", "2026-11-11"), meter(7, "NEW", "2026-11-12")], + periods: [], + restarts: [ + { warehouse_id: 7, warehouse_name: "AOLD", day: "2026-11-12", gap_seconds: 30, bounces: 1 }, + { warehouse_id: 7, warehouse_name: "NEW", day: "2026-11-12", gap_seconds: 30, bounces: 2 }, + ], + queries: [ + { warehouse_id: 7, warehouse_name: "AOLD", query_count: 10, sub_five_second_queries: 10, sub_five_second_credits: 1 }, + { warehouse_id: 7, warehouse_name: "NEW", query_count: 990, sub_five_second_queries: 0, sub_five_second_credits: 0.5 }, + ], + }) + expect(facts[0]).toMatchObject({ query_count: 1000, sub_five_second_queries: 10, sub_five_second_credits: 1.5 }) + expect(facts[0]!.restart_days).toEqual([{ day: "2026-11-12", gaps: [{ gap_seconds: 30, bounces: 3 }] }]) + }) + + test("two warehouses that swapped names keep their own usage and distinct ids", () => { + const { facts } = buildFacts({ + warehouses: [show({ name: "BLUE" }), show({ name: "GREEN" })], + metering: [meter(7, "BLUE", "2026-11-11", 1), meter(8, "GREEN", "2026-11-11", 2), meter(7, "GREEN", "2026-11-12", 3), meter(8, "BLUE", "2026-11-12", 4)], + ...none, + }) + expect(facts.map((one) => one.warehouse_id)).toEqual(["8", "7"]) + expect(facts[0]!.usage_days.map((day) => day.credits_used)).toEqual([2, 4]) + expect(facts[1]!.usage_days.map((day) => day.credits_used)).toEqual([1, 3]) + }) + + test("a new unused warehouse on a renamed one's old name gets none of its usage", () => { + const { facts } = buildFacts({ + warehouses: [show({ name: "ETL" }), show({ name: "ETL_OLD" })], + metering: [meter(7, "ETL", "2026-11-11"), meter(7, "ETL_OLD", "2026-11-12")], + ...none, + }) + expect(facts.map((one) => one.warehouse_id)).toEqual(["ETL", "7"]) + expect(facts[0]!.usage_days).toEqual([]) + expect(facts[1]!.usage_days.length).toBe(2) + }) + + test("usage that ended before the listed warehouse was created belongs to its predecessor", () => { + const rows = { metering: [meter(100, "BI_WH", "2026-11-12")], ...none } + const recreated = buildFacts({ warehouses: [show({ created_on: new Date("2026-11-15T08:00:00Z") })], ...rows }) + expect(recreated.facts[0]).toMatchObject({ warehouse_id: "BI_WH", usage_days: [] }) + expect(recreated.unlisted).toEqual([]) + const older = buildFacts({ warehouses: [show({ created_on: "2026-01-01T00:00:00Z" })], ...rows }) + expect(older.facts[0]!.warehouse_id).toBe("100") + // A creation time that cannot be read changes nothing. + expect(buildFacts({ warehouses: [show({ created_on: "not a time" })], ...rows }).facts[0]!.warehouse_id).toBe("100") + }) + + test("a warehouse that billed credits but cannot be listed is named", () => { + const { facts, unlisted } = buildFacts({ + warehouses: [show()], + metering: [{ warehouse_id: 3, warehouse_name: "SECRET_WH", usage_day: "2026-11-11", idle_day: null, credits_used: 1 }], + periods: [], + restarts: [], + queries: [], + }) + expect(unlisted).toEqual(["SECRET_WH"]) + expect(facts.map((one) => one.warehouse_name)).toEqual(["BI_WH"]) + }) +}) + +describe("the price of a credit", () => { + test("is the rate sheet's in dollars, otherwise list price", () => { + expect(creditPrice({ effective_rate: "2.40", currency: "USD", rate_date: "2026-11-17" })).toEqual({ per_credit: 2.4, source: "rate_sheet", as_of: "2026-11-17" }) + for (const rate of [undefined, { effective_rate: 2.4, currency: "EUR" }, { effective_rate: 0, currency: "USD" }]) { + expect(creditPrice(rate)).toEqual({ per_credit: LIST_PRICE_PER_CREDIT, source: "list_price" }) + } + }) +}) + +describe("snowflake savings handler over a stubbed connection", () => { + const NOW = Date.parse("2026-11-18T10:00:00Z") + const table = (rows: Record[], truncated = false) => ({ + columns: Object.keys(rows[0] ?? {}), + rows: rows.map((row) => Object.values(row)), + truncated, + }) + const SHOW = [{ name: "BI_WH", size: "Large", type: "STANDARD", min_cluster_count: 1, max_cluster_count: 1 }] + const METERING = [{ warehouse_id: 7, warehouse_name: "BI_WH", usage_day: "2026-11-11", idle_day: "2026-11-11", credits_used: 14 }] + const RATE = [{ effective_rate: 2.5, currency: "USD", rate_date: "2026-11-17" }] + const answers = (sql: string) => + sql === SNOWFLAKE_WAREHOUSES_SQL ? table(SHOW) : sql === SNOWFLAKE_METERING_FACTS_SQL ? table(METERING) : sql === SNOWFLAKE_RATE_SQL ? table(RATE) : table([]) + let calls: { sql: string; limit: number; binds: unknown[]; timeout?: number; clientTimeout?: number; signal?: AbortSignal }[] + let sent: any + + beforeEach(() => { + calls = [] + sent = undefined + Registry.setConfigs({ sf: { type: "snowflake", account: "a", user: "u", password: "p", warehouse: "BI_WH" } } as any) + }) + afterEach(() => Registry.reset()) + + const rules = (input: string) => { + sent = JSON.parse(input) + return { lookback_days: 7, price_per_credit: sent.price_per_credit, findings: [], total: { monthly_min: 0, monthly_max: 0, resources: 0 }, what_if: [] } + } + const connect = (answer: (sql: string) => unknown) => + spyOn(Registry, "get").mockResolvedValue({ + execute: async (sql: string, limit: number, binds: unknown[], options?: { cancelAfterMs?: number; queryTimeoutMs?: number; signal?: AbortSignal }) => { + calls.push({ sql, limit, binds, timeout: options?.cancelAfterMs, clientTimeout: options?.queryTimeoutMs, signal: options?.signal }) + return answer(sql) + }, + } as any) + + test("every query is bound to the window end and gets what is left of the 9 minutes; the facts and rate reach core", async () => { + const spy = connect(answers) + const result = await snowflakeSavings({ warehouse: "sf", now_ms: NOW }, rules) + spy.mockRestore() + expect(result.success).toBe(true) + expect(result.window).toEqual({ start: "2026-11-11", end: "2026-11-18" }) + expect(calls.map((one) => one.sql)).toEqual([ + SNOWFLAKE_WAREHOUSES_SQL, + SNOWFLAKE_RATE_SQL, + SNOWFLAKE_METERING_FACTS_SQL, + SNOWFLAKE_PERIOD_FACTS_SQL, + SNOWFLAKE_RESTART_FACTS_SQL, + SNOWFLAKE_QUERY_FACTS_SQL, + ]) + for (const one of calls) { + expect(one.binds).toEqual(Array(one.sql.split("?").length - 1).fill("2026-11-18")) + expect(one.timeout).toBeGreaterThan(0) + expect(one.timeout).toBeLessThanOrEqual(SAVINGS_RUN_LIMIT_MS) + expect(one.clientTimeout).toBeUndefined() + // SHOW takes a LIMIT of at most 10,000, and the driver asks for one row more than this. + expect(one.limit).toBe(one.sql === SNOWFLAKE_WAREHOUSES_SQL ? SHOW_ROW_LIMIT : ROW_LIMIT) + } + expect(SHOW_ROW_LIMIT + 1).toBeLessThanOrEqual(10_000) + expect(SAVINGS_RUN_LIMIT_MS).toBe(540_000) + expect(sent).toMatchObject({ lookback_days: 7, price_per_credit: 2.5 }) + expect(sent.warehouses[0]).toMatchObject({ warehouse_id: "7", warehouse_name: "BI_WH", warehouse_size: "Large" }) + expect(result.price).toEqual({ per_credit: 2.5, source: "rate_sheet", as_of: "2026-11-17" }) + expect(result.ran_on).toBe("BI_WH") + expect(result.warehouses).toEqual([{ id: "7", name: "BI_WH", type: "STANDARD", size: "Large", window_cost: 35 }]) + }) + + test("a rate sheet the role cannot read falls back to the stated list price and skips nothing", async () => { + const spy = connect((sql) => { + if (sql === SNOWFLAKE_RATE_SQL) throw new Error("Object does not exist or not authorized") + return answers(sql) + }) + const result = await snowflakeSavings({ warehouse: "sf", now_ms: NOW }, rules) + spy.mockRestore() + expect(result.success).toBe(true) + expect(sent.price_per_credit).toBe(LIST_PRICE_PER_CREDIT) + expect(result.price).toEqual({ per_credit: LIST_PRICE_PER_CREDIT, source: "list_price" }) + expect(result.skipped).toEqual([]) + }) + + test("a view the role cannot read skips only the checks that need it", async () => { + const spy = connect((sql) => { + if (sql === SNOWFLAKE_RESTART_FACTS_SQL) throw new Error("not authorized\nstack") + return answers(sql) + }) + const result = await snowflakeSavings({ warehouse: "sf", now_ms: NOW }, rules) + spy.mockRestore() + expect(result.success).toBe(true) + expect(result.skipped).toEqual([{ check: "Frequent Restart Overcharge", reason: "not authorized" }]) + }) + + test("when none of the three usage queries can run, the check fails instead of reporting no savings", async () => { + const spy = connect((sql) => { + if ([SNOWFLAKE_PERIOD_FACTS_SQL, SNOWFLAKE_RESTART_FACTS_SQL, SNOWFLAKE_QUERY_FACTS_SQL].includes(sql)) throw new Error("not authorized") + return answers(sql) + }) + const result = await snowflakeSavings({ warehouse: "sf", now_ms: NOW }, rules) + spy.mockRestore() + expect(result).toMatchObject({ success: false, error: "No check could run. not authorized" }) + expect(sent).toBeUndefined() + }) + + test("billing that cannot be read fails the check: nothing could be priced", async () => { + const spy = connect((sql) => { + if (sql === SNOWFLAKE_METERING_FACTS_SQL) throw new Error("Insufficient privileges") + return answers(sql) + }) + const result = await snowflakeSavings({ warehouse: "sf", now_ms: NOW }, rules) + spy.mockRestore() + expect(result).toMatchObject({ success: false, error: "Insufficient privileges" }) + expect(sent).toBeUndefined() + }) + + test("a role that can list no warehouse gets a plain failure before any usage query", async () => { + const spy = connect(() => table([])) + const result = await snowflakeSavings({ warehouse: "sf", now_ms: NOW }, rules) + spy.mockRestore() + expect(result.success).toBe(false) + expect(result.error).toContain("SHOW WAREHOUSES lists no warehouse") + expect(calls.length).toBe(1) + }) + + test("billed warehouses the role cannot list are named as not checked", async () => { + const spy = connect((sql) => + sql === SNOWFLAKE_METERING_FACTS_SQL + ? table([...METERING, { warehouse_id: 3, warehouse_name: "SECRET_WH", usage_day: "2026-11-11", idle_day: null, credits_used: 1 }]) + : answers(sql), + ) + const result = await snowflakeSavings({ warehouse: "sf", now_ms: NOW }, rules) + spy.mockRestore() + expect(result.skipped).toEqual([ + { check: "1 warehouse that billed credits", reason: "SHOW WAREHOUSES does not list SECRET_WH: dropped, renamed, or not visible to the connection's role" }, + ]) + }) + + test("a query past its limit fails with a plain reason, or skips an optional check", async () => { + const timeout = (cancelled: boolean) => Object.assign(new Error("QUERY_TIMEOUT: snowflake query exceeded"), { code: "QUERY_TIMEOUT", cancelled }) + let spy = connect((sql) => { + if (sql === SNOWFLAKE_METERING_FACTS_SQL) throw timeout(true) + return answers(sql) + }) + let result = await snowflakeSavings({ warehouse: "sf", now_ms: NOW }, rules) + spy.mockRestore() + expect(result.error).toBe("The metering query ran past the check's 9-minute limit and was cancelled on the warehouse.") + + spy = connect((sql) => { + if (sql === SNOWFLAKE_QUERY_FACTS_SQL) throw timeout(false) + return answers(sql) + }) + result = await snowflakeSavings({ warehouse: "sf", now_ms: NOW }, rules) + spy.mockRestore() + expect(result.success).toBe(true) + expect(result.skipped).toEqual([ + { check: "The short-query signal of Downsize Warehouse", reason: "The query history query ran past the check's 9-minute limit and may still be running on the warehouse." }, + ]) + }) + + test("each query gets only what the earlier ones left; with nothing left it is not started", async () => { + let now = 1_000_000 + const clock = spyOn(Date, "now").mockImplementation(() => now) + const spy = connect((sql) => { + // The metering query uses up two minutes, and the period query all the rest. + if (sql === SNOWFLAKE_METERING_FACTS_SQL) now += 120_000 + if (sql === SNOWFLAKE_PERIOD_FACTS_SQL) now += SAVINGS_RUN_LIMIT_MS + return answers(sql) + }) + const result = await snowflakeSavings({ warehouse: "sf", now_ms: NOW }, rules) + spy.mockRestore() + clock.mockRestore() + expect(calls.map((one) => one.timeout)).toEqual([ + SAVINGS_RUN_LIMIT_MS, + SAVINGS_RUN_LIMIT_MS, + SAVINGS_RUN_LIMIT_MS, + SAVINGS_RUN_LIMIT_MS - 120_000, + ]) + expect(result.success).toBe(true) + expect(result.skipped).toEqual([ + { check: "Frequent Restart Overcharge", reason: "The restart events query was not started: it would run past the check's 9-minute limit." }, + { check: "The short-query signal of Downsize Warehouse", reason: "The query history query was not started: it would run past the check's 9-minute limit." }, + ]) + }) + + test("reaching the limit before the required billing query fails the check", async () => { + let now = 1_000_000 + const clock = spyOn(Date, "now").mockImplementation(() => now) + const spy = connect((sql) => { + if (sql === SNOWFLAKE_RATE_SQL) now += SAVINGS_RUN_LIMIT_MS + return answers(sql) + }) + const result = await snowflakeSavings({ warehouse: "sf", now_ms: NOW }, rules) + spy.mockRestore() + clock.mockRestore() + expect(result).toMatchObject({ success: false, error: "The metering query was not started: it would run past the check's 9-minute limit." }) + }) + + test("stopping the session cancels the running query and starts no other", async () => { + const controller = new AbortController() + const spy = connect((sql) => { + if (sql === SNOWFLAKE_PERIOD_FACTS_SQL) { + controller.abort(new Error("session stopped")) + throw new Error("session stopped") + } + return answers(sql) + }) + const result = await snowflakeSavings({ warehouse: "sf", now_ms: NOW }, rules, { signal: controller.signal }) + spy.mockRestore() + expect(result).toMatchObject({ success: false, error: "session stopped" }) + expect(result.skipped).toBeUndefined() + expect(calls.at(-1)!.sql).toBe(SNOWFLAKE_PERIOD_FACTS_SQL) + expect(sent).toBeUndefined() + // Each query got the session's signal, so the driver cancels it on Snowflake. + for (const one of calls) expect(one.signal).toBe(controller.signal) + }) + + test("connecting counts toward the limit: a connection that never comes up ends the check in time", async () => { + // A 30 ms session budget stands in for the last moments of the 9 minutes. + const budget = spyOn(RunBudget, "queryDeadline").mockReturnValue(Date.now() + 30) + const get = spyOn(Registry, "get").mockReturnValue(new Promise(() => {}) as any) + const result = await snowflakeSavings({ warehouse: "sf", now_ms: NOW }, rules, { sessionID: "s1" }) + get.mockRestore() + budget.mockRestore() + expect(result).toMatchObject({ success: false, error: "Connecting to the warehouse took longer than the run's time budget." }) + }) + + test("stopping the session while connecting ends the check at once", async () => { + const controller = new AbortController() + const get = spyOn(Registry, "get").mockReturnValue(new Promise(() => {}) as any) + const pending = snowflakeSavings({ warehouse: "sf", now_ms: NOW }, rules, { signal: controller.signal }) + controller.abort(new Error("session stopped")) + const result = await pending + get.mockRestore() + expect(result).toMatchObject({ success: false, error: "session stopped" }) + }) + + test("a rate query that may still be running is named, and list price is used", async () => { + const timeout = Object.assign(new Error("QUERY_TIMEOUT"), { code: "QUERY_TIMEOUT", cancelled: false }) + const spy = connect((sql) => { + if (sql === SNOWFLAKE_RATE_SQL) throw timeout + return answers(sql) + }) + const result = await snowflakeSavings({ warehouse: "sf", now_ms: NOW }, rules) + spy.mockRestore() + expect(result.price?.source).toBe("list_price") + expect(result.skipped).toEqual([ + { check: "The account's credit rate", reason: "The credit rate query ran past the check's 9-minute limit and may still be running on the warehouse." }, + ]) + }) + + test("a session time budget shorter than 9 minutes is the limit, and is named when it stops a query", async () => { + const budget = spyOn(RunBudget, "queryDeadline").mockReturnValue(Date.now() + 60_000) + const timeout = Object.assign(new Error("QUERY_TIMEOUT"), { code: "QUERY_TIMEOUT", cancelled: true }) + const spy = connect((sql) => { + if (sql === SNOWFLAKE_METERING_FACTS_SQL) throw timeout + return answers(sql) + }) + const result = await snowflakeSavings({ warehouse: "sf", now_ms: NOW }, rules, { sessionID: "s1" }) + spy.mockRestore() + budget.mockRestore() + for (const one of calls) expect(one.timeout).toBeLessThanOrEqual(60_000) + expect(result.error).toBe("The metering query ran past the run's time budget and was cancelled on the warehouse.") + }) + + test("a stop during the rate lookup is not mistaken for an unreadable rate sheet", async () => { + const controller = new AbortController() + const spy = connect((sql) => { + if (sql === SNOWFLAKE_RATE_SQL) { + controller.abort(new Error("session stopped")) + throw new Error("session stopped") + } + return answers(sql) + }) + const result = await snowflakeSavings({ warehouse: "sf", now_ms: NOW }, rules, { signal: controller.signal }) + spy.mockRestore() + expect(result).toMatchObject({ success: false, error: "session stopped" }) + expect(calls.at(-1)!.sql).toBe(SNOWFLAKE_RATE_SQL) + }) + + test("a query that fills the row limit is named", async () => { + const spy = connect((sql) => (sql === SNOWFLAKE_PERIOD_FACTS_SQL ? table([], true) : answers(sql))) + const result = await snowflakeSavings({ warehouse: "sf", now_ms: NOW }, rules) + spy.mockRestore() + expect(result.limited).toEqual(["active periods"]) + }) + + test("with no Snowflake connection it says how to add one", async () => { + Registry.setConfigs({}) + const result = await snowflakeSavings({}, rules) + expect(result.success).toBe(false) + expect(result.error).toContain("warehouse_add") + }) + + test("a core without the rules is refused before any query", async () => { + const spy = connect(answers) + const result = await snowflakeSavings({ warehouse: "sf", now_ms: NOW }, null) + spy.mockRestore() + expect(result.error).toContain("no Snowflake savings rules") + expect(calls).toEqual([]) + }) + + test("in assessment mode the rules come with core's own telemetry switched off", () => { + const saved = process.env["ALTIMATE_ASSESSMENT_MODE"] + const calls: unknown[][] = [] + const lib = { snowflakeSavings: () => ({}), initSdk: (...args: unknown[]) => void calls.push(args) } + try { + process.env["ALTIMATE_ASSESSMENT_MODE"] = "true" + expect(rulesEngine(lib)).toBe(lib.snowflakeSavings as never) + expect(calls).toEqual([["", "", "", false, false]]) + expect(rulesEngine({ initSdk: lib.initSdk })).toBeNull() + delete process.env["ALTIMATE_ASSESSMENT_MODE"] + rulesEngine(lib) + expect(calls.length).toBe(1) + } finally { + if (saved === undefined) delete process.env["ALTIMATE_ASSESSMENT_MODE"] + else process.env["ALTIMATE_ASSESSMENT_MODE"] = saved + } + }) +}) From ade5a6b5e6bbbeb5105f6ef3330ad2bfa302f2c2 Mon Sep 17 00:00:00 2001 From: Shreyas Telkar Date: Wed, 18 Nov 2026 15:12:11 -0800 Subject: [PATCH 26/34] fix: Snowflake savings counts restart gaps in whole elapsed seconds Also: the rate query takes one price above zero, a stop cancels every statement sent, unused warehouses get ids that cannot collide, account names stay on one line, and the driver egress test no longer needs the core rules. Co-Authored-By: Claude Opus 5.5 --- .../data-engineering/tools/finops-tools.md | 2 +- packages/drivers/src/snowflake.ts | 17 ++-- .../drivers/test/snowflake-deadline.test.ts | 3 +- .../native/finops/snowflake-savings.ts | 23 ++++-- .../tools/finops-databricks-savings.ts | 2 +- .../tools/finops-snowflake-savings.ts | 17 ++-- .../altimate/assessment-zero-egress.test.ts | 3 +- .../altimate/finops-snowflake-savings.test.ts | 82 ++++++++++++++++++- 8 files changed, 119 insertions(+), 30 deletions(-) diff --git a/docs/docs/data-engineering/tools/finops-tools.md b/docs/docs/data-engineering/tools/finops-tools.md index ac07b529b4..575c4f057c 100644 --- a/docs/docs/data-engineering/tools/finops-tools.md +++ b/docs/docs/data-engineering/tools/finops-tools.md @@ -211,7 +211,7 @@ Classic (sql warehouse, workspace 525779153077873) · Migrate Classic/Pro to Ser ## finops_snowflake_savings -Check a Snowflake account for savings with Altimate's rules. It lists warehouses with `SHOW WAREHOUSES` and reads seven days of `SNOWFLAKE.ACCOUNT_USAGE` (`WAREHOUSE_METERING_HISTORY`, `WAREHOUSE_EVENTS_HISTORY`, `QUERY_HISTORY`), read-only, on the connection's warehouse. The role needs `IMPORTED PRIVILEGES` on the `SNOWFLAKE` database, and sees only the warehouses it has access to; warehouses that billed credits but cannot be listed are named as not checked. +Check a Snowflake account for savings with Altimate's rules. It lists warehouses with `SHOW WAREHOUSES` and, to report on seven days, reads up to eleven days of `SNOWFLAKE.ACCOUNT_USAGE` (`WAREHOUSE_METERING_HISTORY`, `WAREHOUSE_EVENTS_HISTORY`, `QUERY_HISTORY`), read-only, on the connection's warehouse. The role needs `IMPORTED PRIVILEGES` on the `SNOWFLAKE` database, and sees only the warehouses it has access to; warehouses that billed credits but cannot be listed are named as not checked. - **High Idle Time:** on days a warehouse sat idle for more than 40% of its active time, the idle share of that day's cost. - **Downsize Warehouse:** nearly all queries finish within 5 seconds, or nearly all active periods last 80 seconds or less, so one size smaller would do. diff --git a/packages/drivers/src/snowflake.ts b/packages/drivers/src/snowflake.ts index 384ad2c526..52334c2314 100644 --- a/packages/drivers/src/snowflake.ts +++ b/packages/drivers/src/snowflake.ts @@ -58,7 +58,9 @@ export function suppressSnowflakeLogging(snowflake: any): void { /** * In assessment mode nothing but the warehouse may be contacted. When it loads, snowflake-sdk otherwise probes - * the AWS, Azure and GCP metadata and identity services to report the platform it runs on. + * the AWS, Azure and GCP metadata and identity services to report the platform it runs on. Its certificate + * revocation requests need no switch here: they start from an HTTPS agent that Bun, which runs the packaged + * CLI, does not use. */ export function keepSdkOffCloudMetadata(env: NodeJS.ProcessEnv = process.env): void { const mode = env.ALTIMATE_ASSESSMENT_MODE?.trim().toLowerCase() @@ -68,8 +70,11 @@ export function keepSdkOffCloudMetadata(env: NodeJS.ProcessEnv = process.env): v /** How long to wait for Snowflake to acknowledge a cancel before reporting the statement as possibly still running. */ const CANCEL_WAIT_MS = 30_000 -/** How a caller's `cancelAfterMs` reaches the statement it covers: why it was stopped, and the statement sent. */ -type StatementStop = { stopped?: "deadline" | "aborted"; statement?: any } +/** + * How a caller's `cancelAfterMs` reaches what it covers: why it was stopped, and every statement sent. A read + * retried after a closed-session error leaves two, and the first may still be running. + */ +type StatementStop = { stopped?: "deadline" | "aborted"; statements: any[] } /** * Errors meaning the session is gone: 407002 is raised client-side for any @@ -572,7 +577,7 @@ export async function connect( } if (binds && binds.length > 0) options.binds = binds const statement = conn.execute(options) - if (stop) stop.statement = statement + stop?.statements.push(statement) }) } @@ -602,7 +607,7 @@ export async function connect( signal?: AbortSignal, ): Promise<{ columns: string[]; rows: any[][] }> { signal?.throwIfAborted() - const stop: StatementStop = {} + const stop: StatementStop = { statements: [] } const STOPPED = Symbol("stopped") let wake: () => void = () => {} const stopping = new Promise((resolve) => (wake = () => resolve(STOPPED))) @@ -623,7 +628,7 @@ export async function connect( } if (outcome !== STOPPED) return outcome work.catch(() => {}) - const cancelled = stop.statement ? await cancelStatement(stop.statement) : true + const cancelled = (await Promise.all(stop.statements.map(cancelStatement))).every(Boolean) if (stop.stopped === "aborted") throw signal!.reason throw new QueryTimeoutError({ engine: "snowflake", timeoutMs: deadlineMs, cancelled }) } diff --git a/packages/drivers/test/snowflake-deadline.test.ts b/packages/drivers/test/snowflake-deadline.test.ts index 5c6e0dea9e..ba1a3d25b2 100644 --- a/packages/drivers/test/snowflake-deadline.test.ts +++ b/packages/drivers/test/snowflake-deadline.test.ts @@ -184,7 +184,8 @@ describe("Snowflake server-side cancel", () => { const error = await db.execute("SELECT 1", 10, [], { cancelAfterMs: 40 }).catch((e) => e) expect(isQueryTimeoutError(error)).toBe(true) expect(sent).toEqual(["SELECT 1 LIMIT 11", "SELECT 1 LIMIT 11"]) - expect(cancels).toBe(1) + // The first copy may still be running, so both are cancelled. + expect(cancels).toBe(2) }) test("a statement whose session closed may still be running, so the deadline tries to cancel it", async () => { diff --git a/packages/opencode/src/altimate/native/finops/snowflake-savings.ts b/packages/opencode/src/altimate/native/finops/snowflake-savings.ts index e703db1ead..fb0d3e0739 100644 --- a/packages/opencode/src/altimate/native/finops/snowflake-savings.ts +++ b/packages/opencode/src/altimate/native/finops/snowflake-savings.ts @@ -143,7 +143,7 @@ FROM per_period GROUP BY 1, 2, 3, 4, 5 ORDER BY 2, 3, 4, 5 /** * Per warehouse and day: how many times it resumed, suspended and resumed again less than a minute after - * the first resume, by the seconds between the two resumes, counted as for active periods. The event log + * the first resume, by the whole seconds elapsed between the two resumes. The event log * repeats each warehouse-level event, so events are made distinct first. */ export const SNOWFLAKE_RESTART_FACTS_SQL = ` @@ -164,11 +164,11 @@ s AS ( FROM e ) SELECT warehouse_id, warehouse_name, TO_VARCHAR(ts::DATE, 'YYYY-MM-DD') AS day, - DATEDIFF('second', ts, t4) AS gap_seconds, COUNT(*) AS bounces + TRUNC(DATEDIFF('millisecond', ts, t4) / 1000) AS gap_seconds, COUNT(*) AS bounces FROM s WHERE event_name = 'RESUME_WAREHOUSE' AND n1 = 'WAREHOUSE_CONSISTENT' AND n2 = 'SUSPEND_WAREHOUSE' AND n3 = 'WAREHOUSE_CONSISTENT' AND n4 = 'RESUME_WAREHOUSE' - AND DATEDIFF('second', ts, t4) < 60 + AND DATEDIFF('millisecond', ts, t4) < 60000 GROUP BY 1, 2, 3, 4 ORDER BY 2, 3, 4 ` @@ -226,12 +226,13 @@ WHERE q.et >= DATEADD(day, -7, ${WINDOW_END_UTC}) AND q.et < ${WINDOW_END_UTC} GROUP BY 1, 2 ORDER BY 2 ` -/** The account's contracted price of a warehouse credit, latest before the window's end. */ +/** The account's contracted price of a warehouse credit: the latest above zero before the window's end, the lowest if that date has several. */ export const SNOWFLAKE_RATE_SQL = ` SELECT EFFECTIVE_RATE AS effective_rate, CURRENCY AS currency, TO_VARCHAR(DATE, 'YYYY-MM-DD') AS rate_date FROM SNOWFLAKE.ORGANIZATION_USAGE.RATE_SHEET_DAILY WHERE ACCOUNT_NAME = CURRENT_ACCOUNT_NAME() AND SERVICE_TYPE = 'WAREHOUSE_METERING' AND USAGE_TYPE = 'compute' AND DATE < ${WINDOW_END_UTC}::DATE -ORDER BY DATE DESC LIMIT 1 + AND EFFECTIVE_RATE > 0 +ORDER BY DATE DESC, EFFECTIVE_RATE LIMIT 1 ` /** Core's rules: a `SnowflakeSavingsInput` as JSON in, the report out. */ @@ -393,7 +394,8 @@ export function buildFacts(rows: FactRows) { query.credits += num(one.sub_five_second_credits) } return { - warehouse_id: id ? String(id) : name, + // Prefixed so an unused warehouse named like another's numeric id cannot collide with it. + warehouse_id: id ? String(id) : `unused:${name}`, warehouse_name: name, warehouse_size: row.size == null ? null : String(row.size), warehouse_type: row.type == null ? null : String(row.type), @@ -451,6 +453,8 @@ export async function snowflakeSavings( // Connecting can wait on the network or a sign-in: it counts toward the limit and yields to a stop. const beforeStop = async (work: Promise, what: string): Promise => { + // A stop can come first, and a later failure of `work` must not go unhandled. + work.catch(() => {}) signal?.throwIfAborted() let timer: ReturnType | undefined let onAbort: (() => void) | undefined @@ -468,7 +472,6 @@ export async function snowflakeSavings( } finally { clearTimeout(timer) if (onAbort) signal?.removeEventListener("abort", onAbort) - work.catch(() => {}) } } @@ -527,7 +530,11 @@ export async function snowflakeSavings( const queries = await optional("The short-query signal of Downsize Warehouse", "query history", SNOWFLAKE_QUERY_FACTS_SQL) // All four checks rest on these three queries: with none of them, nothing was assessed. - if (checksMissing === 3) return failed(`No check could run. ${skipped.at(-1)!.reason}`) + if (checksMissing === 3) { + // A query that may still be running and billing matters more than the ones after it. + const reason = skipped.find((one) => one.reason.includes("may still be running")) ?? skipped.at(-1)! + return failed(`No check could run. ${reason.reason}`) + } const price = creditPrice(rate[0]) const { facts, unlisted } = buildFacts({ warehouses, metering, periods, restarts, queries }) diff --git a/packages/opencode/src/altimate/tools/finops-databricks-savings.ts b/packages/opencode/src/altimate/tools/finops-databricks-savings.ts index fa1e10ca06..480c6df848 100644 --- a/packages/opencode/src/altimate/tools/finops-databricks-savings.ts +++ b/packages/opencode/src/altimate/tools/finops-databricks-savings.ts @@ -47,7 +47,7 @@ const tenths = (value: unknown) => Number(value) < 0.05 ? "under 0.1" : Number(value).toLocaleString("en-US", { minimumFractionDigits: 1, maximumFractionDigits: 1 }) /** Free text from the workspace on one line, with control and invisible characters gone. */ -const oneLine = (value: unknown) => +export const oneLine = (value: unknown) => String(value ?? "") .replace(/\p{Cf}/gu, "") .replace(/[\s\p{Cc}]+/gu, " ") diff --git a/packages/opencode/src/altimate/tools/finops-snowflake-savings.ts b/packages/opencode/src/altimate/tools/finops-snowflake-savings.ts index 339c87f3fa..a748cd6c38 100644 --- a/packages/opencode/src/altimate/tools/finops-snowflake-savings.ts +++ b/packages/opencode/src/altimate/tools/finops-snowflake-savings.ts @@ -3,7 +3,7 @@ import z from "zod" import { Tool } from "../../tool/tool" import { Dispatcher } from "../native" import type { SnowflakeSavingsFinding, SnowflakeSavingsReport, SnowflakeSavingsResult } from "../native/types" -import { dollars, range } from "./finops-databricks-savings" +import { dollars, oneLine, range } from "./finops-databricks-savings" /** One shape for every outcome, so a reader of the metadata never guesses which fields exist. */ type SavingsMetadata = { @@ -73,10 +73,10 @@ function priceLine(result: SnowflakeSavingsResult): string { : `$${price.per_credit.toFixed(2)} a credit, list price (no dollar rate could be read from the account's rate sheet)` } -function headline(report: SnowflakeSavingsReport): string { +function headline(report: SnowflakeSavingsReport, partial = false): string { const { monthly_min, monthly_max, resources } = report.total if (resources > 0) return `${range(monthly_min, monthly_max)} a month on ${resources} warehouse${resources === 1 ? "" : "s"}` - if (report.findings.length === 0) return "no savings found" + if (report.findings.length === 0) return partial ? "no savings found in what could be checked" : "no savings found" return `${report.findings.length} finding${report.findings.length === 1 ? "" : "s"}, no dollar estimate` } @@ -87,7 +87,8 @@ export function formatSavings(result: SnowflakeSavingsResult): string { const lines: string[] = [ `Snowflake savings: ${checked} warehouse${checked === 1 ? "" : "s"} checked, the ${report.lookback_days} days to ${result.window.end}, at ${priceLine(result)}.`, ] - for (const one of result.skipped ?? []) lines.push(`Not checked: ${one.check} (${one.reason})`) + // Warehouse names come from the account and can hold line breaks. + for (const one of result.skipped ?? []) lines.push(`Not checked: ${oneLine(one.check)} (${oneLine(one.reason)})`) for (const what of result.limited ?? []) lines.push(`Only part of the ${what} could be read; the rest was not checked.`) if (report.findings.length === 0) { lines.push(result.skipped?.length || result.limited?.length ? "No savings found in what could be checked." : "No savings found.") @@ -97,7 +98,7 @@ export function formatSavings(result: SnowflakeSavingsResult): string { lines.push("") for (const finding of report.findings) { const amount = finding.monthly_max > 0 ? `${range(finding.monthly_min, finding.monthly_max)} a month` : "no dollar estimate" - lines.push(`${finding.resource_name} (warehouse) · ${finding.title} · ${amount} (${finding.severity})`) + lines.push(`${oneLine(finding.resource_name) || finding.resource_id} (warehouse) · ${finding.title} · ${amount} (${finding.severity})`) for (const line of evidenceLines(finding)) lines.push(` ${line}`) } return lines.join("\n") @@ -105,7 +106,7 @@ export function formatSavings(result: SnowflakeSavingsResult): string { export const FinopsSnowflakeSavingsTool = Tool.define("finops_snowflake_savings", { description: - "Check a Snowflake account for savings with Altimate's rules: warehouses that sit idle, could run one size smaller, keep an unneeded extra cluster, or restart so often they are billed twice. Read-only: SHOW WAREHOUSES plus five queries over seven days of SNOWFLAKE.ACCOUNT_USAGE and the rate sheet, on the connection's warehouse, which Snowflake resumes if it is suspended and auto-resume is on. Returns each finding with its monthly dollar range and evidence. Ask the user before running it, and pass confirm: true once they agree; without it nothing runs.", + "Check a Snowflake account for savings with Altimate's rules: warehouses that sit idle, could run one size smaller, keep an unneeded extra cluster, or restart so often they are billed twice. Read-only: SHOW WAREHOUSES plus five queries on the connection's warehouse: four read up to eleven days of SNOWFLAKE.ACCOUNT_USAGE, to report on seven, and one reads the rate sheet. Snowflake resumes that warehouse if it is suspended and auto-resume is on. Returns each finding with its monthly dollar range and evidence. Ask the user before running it, and pass confirm: true once they agree; without it nothing runs.", parameters: z.object({ warehouse: z .string() @@ -125,7 +126,7 @@ export const FinopsSnowflakeSavingsTool = Tool.define("finops_snowflake_savings" title: "Snowflake Savings: not run, needs the user's go-ahead", metadata: { success: true, needs_confirmation: true } as SavingsMetadata, output: - "Not run. The Snowflake savings check runs six queries over seven days of account usage; five use the connection's warehouse, " + + "Not run. The Snowflake savings check runs six queries that read up to eleven days of account usage, to report on seven; five use the connection's warehouse, " + "which Snowflake resumes if it is suspended and auto-resume is on, and which bills while it runs. It stops after 9 minutes, cancelling any query still running. " + "Ask the user whether to run it; if they agree, call this tool again with confirm: true.", } @@ -145,7 +146,7 @@ export const FinopsSnowflakeSavingsTool = Tool.define("finops_snowflake_savings" } } return { - title: `Snowflake Savings: ${headline(result.report)}`, + title: `Snowflake Savings: ${headline(result.report, Boolean(result.skipped?.length || result.limited?.length))}`, metadata: { success: true, window: result.window, diff --git a/packages/opencode/test/altimate/assessment-zero-egress.test.ts b/packages/opencode/test/altimate/assessment-zero-egress.test.ts index 92a28edbea..f9ad659737 100644 --- a/packages/opencode/test/altimate/assessment-zero-egress.test.ts +++ b/packages/opencode/test/altimate/assessment-zero-egress.test.ts @@ -162,7 +162,8 @@ describe("assessment mode on the packaged CLI", () => { posixOnly("with the Snowflake driver installed, assessment mode keeps it off the cloud metadata services", async () => { const account = "egress-test" - const args = ["tool", "run", "finops_snowflake_savings", "--args", '{"confirm":true}', "--json", "--no-daemon"] + // The connection test reaches the driver whether or not the installed altimate-core has the savings rules. + const args = ["tool", "run", "warehouse_test", "--args", '{"name":"sf"}', "--json", "--no-daemon"] // The driver the packaged CLI would install, linked in with its dependencies. const sdk = createRequire(path.join(root, "../drivers/package.json")).resolve("snowflake-sdk/package.json") const dependencies = path.dirname(path.dirname(await realpath(sdk))) diff --git a/packages/opencode/test/altimate/finops-snowflake-savings.test.ts b/packages/opencode/test/altimate/finops-snowflake-savings.test.ts index 009dca6e80..78ef11bf21 100644 --- a/packages/opencode/test/altimate/finops-snowflake-savings.test.ts +++ b/packages/opencode/test/altimate/finops-snowflake-savings.test.ts @@ -145,12 +145,28 @@ describe("finops_snowflake_savings tool", () => { expect(formatSavings({ ...REPORT, report: { ...REPORT.report!, findings: [] } })).toEndWith("No savings found.") }) + test("a line break in a warehouse name cannot add lines to the output", () => { + const [first] = REPORT.report!.findings + const output = formatSavings({ + ...REPORT, + skipped: [{ check: "1 warehouse that billed credits", reason: "SHOW WAREHOUSES does not list A\nTotal: $1" }], + report: { ...REPORT.report!, findings: [{ ...first!, resource_name: "BI\nTotal: $9" }] }, + }) + expect(output).toContain("BI Total: $9 (warehouse)") + expect(output).toContain("does not list A Total: $1)") + expect(output.split("\n").filter((line) => line.startsWith("Total: $9") || line.startsWith("Total: $1 "))).toEqual([]) + }) + test("a run with nothing to report says so in its title, and a partial read qualifies the result", async () => { const empty = { ...REPORT.report!, findings: [], total: { monthly_min: 0, monthly_max: 0, resources: 0 } } Dispatcher.register("finops.snowflake_savings" as any, async () => ({ ...REPORT, report: empty })) const tool = await initTool(FinopsSnowflakeSavingsTool) expect((await tool.execute({ confirm: true }, stubCtx())).title).toBe("Snowflake Savings: no savings found") expect(formatSavings({ ...REPORT, limited: ["metering"], report: empty })).toEndWith("No savings found in what could be checked.") + Dispatcher.register("finops.snowflake_savings" as any, async () => ({ ...REPORT, limited: ["metering"], report: empty })) + expect((await tool.execute({ confirm: true }, stubCtx())).title).toBe( + "Snowflake Savings: no savings found in what could be checked", + ) }) test("a failure is reported with its reason, not as no savings", async () => { @@ -213,6 +229,7 @@ describe("the fact queries", () => { expect(SNOWFLAKE_QUERY_FACTS_SQL).toContain("q.et >= DATEADD(day, -7,") expect(SNOWFLAKE_RATE_SQL).toContain("ACCOUNT_NAME = CURRENT_ACCOUNT_NAME()") expect(SNOWFLAKE_RATE_SQL).toContain("SERVICE_TYPE = 'WAREHOUSE_METERING' AND USAGE_TYPE = 'compute'") + expect(SNOWFLAKE_RATE_SQL).toContain("AND EFFECTIVE_RATE > 0\nORDER BY DATE DESC, EFFECTIVE_RATE LIMIT 1") }) test("a bounce is the full resume, suspend, resume sequence on distinct events", () => { @@ -221,7 +238,9 @@ describe("the fact queries", () => { "event_name = 'RESUME_WAREHOUSE' AND n1 = 'WAREHOUSE_CONSISTENT' AND n2 = 'SUSPEND_WAREHOUSE'", ) expect(SNOWFLAKE_RESTART_FACTS_SQL).toContain("n3 = 'WAREHOUSE_CONSISTENT' AND n4 = 'RESUME_WAREHOUSE'") - expect(SNOWFLAKE_RESTART_FACTS_SQL).toContain("DATEDIFF('second', ts, t4) < 60") + expect(SNOWFLAKE_RESTART_FACTS_SQL).toContain("TRUNC(DATEDIFF('millisecond', ts, t4) / 1000) AS gap_seconds") + expect(SNOWFLAKE_RESTART_FACTS_SQL).toContain("DATEDIFF('millisecond', ts, t4) < 60000") + expect(SNOWFLAKE_RESTART_FACTS_SQL).not.toContain("DATEDIFF('second'") }) }) @@ -271,7 +290,7 @@ describe("building the facts", () => { restart_days: [{ day: "2026-11-11", gaps: [{ gap_seconds: 30, bounces: 4 }] }], }) // A listed warehouse with no usage is still an entry, under its name. - expect(facts[1]).toMatchObject({ warehouse_id: "IDLE_WH", usage_days: [], query_count: 0 }) + expect(facts[1]).toMatchObject({ warehouse_id: "unused:IDLE_WH", warehouse_name: "IDLE_WH", usage_days: [], query_count: 0 }) }) test("a billed day with no periods keeps its credits, and a day with periods but no billing has none", () => { @@ -317,6 +336,20 @@ describe("building the facts", () => { expect(facts[0]!.usage_days.map((day) => day.credits_used)).toEqual([1, 2]) }) + test.each([ + ["the old name first", ["ETL", "ETL_NEW"]], + ["the new name first", ["ETL_NEW", "ETL"]], + ])("a warehouse renamed on its last day of usage stays under its listed name, %s", (_, names) => { + const { facts, unlisted } = buildFacts({ + warehouses: [show({ name: "ETL_NEW" })], + metering: names.map((name, at) => meter(7, name, "2026-11-12", at + 1)), + ...none, + }) + expect(unlisted).toEqual([]) + expect(facts[0]).toMatchObject({ warehouse_id: "7", warehouse_name: "ETL_NEW" }) + expect(facts[0]!.usage_days.map((day) => day.credits_used)).toEqual([3]) + }) + test("a rename's query and restart rows are added, not read from one name or repeated", () => { const { facts } = buildFacts({ warehouses: [show({ name: "NEW" })], @@ -346,13 +379,22 @@ describe("building the facts", () => { expect(facts[1]!.usage_days.map((day) => day.credits_used)).toEqual([1, 3]) }) + test("an unused warehouse named like another's id keeps a distinct id", () => { + const { facts } = buildFacts({ + warehouses: [show({ name: "7" }), show({ name: "ETL" })], + metering: [meter(7, "ETL", "2026-11-11")], + ...none, + }) + expect(facts.map((one) => one.warehouse_id)).toEqual(["unused:7", "7"]) + }) + test("a new unused warehouse on a renamed one's old name gets none of its usage", () => { const { facts } = buildFacts({ warehouses: [show({ name: "ETL" }), show({ name: "ETL_OLD" })], metering: [meter(7, "ETL", "2026-11-11"), meter(7, "ETL_OLD", "2026-11-12")], ...none, }) - expect(facts.map((one) => one.warehouse_id)).toEqual(["ETL", "7"]) + expect(facts.map((one) => one.warehouse_id)).toEqual(["unused:ETL", "7"]) expect(facts[0]!.usage_days).toEqual([]) expect(facts[1]!.usage_days.length).toBe(2) }) @@ -360,7 +402,7 @@ describe("building the facts", () => { test("usage that ended before the listed warehouse was created belongs to its predecessor", () => { const rows = { metering: [meter(100, "BI_WH", "2026-11-12")], ...none } const recreated = buildFacts({ warehouses: [show({ created_on: new Date("2026-11-15T08:00:00Z") })], ...rows }) - expect(recreated.facts[0]).toMatchObject({ warehouse_id: "BI_WH", usage_days: [] }) + expect(recreated.facts[0]).toMatchObject({ warehouse_id: "unused:BI_WH", usage_days: [] }) expect(recreated.unlisted).toEqual([]) const older = buildFacts({ warehouses: [show({ created_on: "2026-01-01T00:00:00Z" })], ...rows }) expect(older.facts[0]!.warehouse_id).toBe("100") @@ -479,6 +521,20 @@ describe("snowflake savings handler over a stubbed connection", () => { expect(result.skipped).toEqual([{ check: "Frequent Restart Overcharge", reason: "not authorized" }]) }) + test("when no usage query could run, one that may still be running is the one named", async () => { + const timeout = Object.assign(new Error("QUERY_TIMEOUT"), { code: "QUERY_TIMEOUT", cancelled: false }) + const spy = connect((sql) => { + if (sql === SNOWFLAKE_PERIOD_FACTS_SQL) throw timeout + if ([SNOWFLAKE_RESTART_FACTS_SQL, SNOWFLAKE_QUERY_FACTS_SQL].includes(sql)) throw new Error("not authorized") + return answers(sql) + }) + const result = await snowflakeSavings({ warehouse: "sf", now_ms: NOW }, rules) + spy.mockRestore() + expect(result.success).toBe(false) + expect(result.error).toContain("The active periods query ran past") + expect(result.error).toContain("may still be running on the warehouse") + }) + test("when none of the three usage queries can run, the check fails instead of reporting no savings", async () => { const spy = connect((sql) => { if ([SNOWFLAKE_PERIOD_FACTS_SQL, SNOWFLAKE_RESTART_FACTS_SQL, SNOWFLAKE_QUERY_FACTS_SQL].includes(sql)) throw new Error("not authorized") @@ -622,6 +678,24 @@ describe("snowflake savings handler over a stubbed connection", () => { expect(result).toMatchObject({ success: false, error: "session stopped" }) }) + test("a connection that fails after the session already stopped is not left unhandled", async () => { + const controller = new AbortController() + controller.abort(new Error("session stopped")) + const unhandled: unknown[] = [] + const note = (reason: unknown) => unhandled.push(reason) + process.on("unhandledRejection", note) + let fail!: (reason: Error) => void + const connecting = new Promise((_, reject) => (fail = reject)) + const get = spyOn(Registry, "get").mockReturnValue(connecting as any) + const result = await snowflakeSavings({ warehouse: "sf", now_ms: NOW }, rules, { signal: controller.signal }) + fail(new Error("connection refused")) + await new Promise((resolve) => setTimeout(resolve, 0)) + get.mockRestore() + process.off("unhandledRejection", note) + expect(result).toMatchObject({ success: false, error: "session stopped" }) + expect(unhandled).toEqual([]) + }) + test("a rate query that may still be running is named, and list price is used", async () => { const timeout = Object.assign(new Error("QUERY_TIMEOUT"), { code: "QUERY_TIMEOUT", cancelled: false }) const spy = connect((sql) => { From 89567af610e474d70697c35cb5b42d437ba2b8dc Mon Sep 17 00:00:00 2001 From: Shreyas Telkar Date: Thu, 19 Nov 2026 16:16:16 -0800 Subject: [PATCH 27/34] feat: the Databricks savings check reports how many workspaces it checked `workspaces_checked` counts the distinct workspaces across every warehouse, cluster and job row checked, including workspaces with no finding or no SQL warehouse. Co-Authored-By: Claude Opus 5.5 --- .../native/finops/databricks-savings.ts | 7 ++++++ .../opencode/src/altimate/native/types.ts | 2 ++ .../tools/finops-databricks-savings.ts | 2 ++ .../finops-databricks-savings.test.ts | 25 +++++++++++++++++++ 4 files changed, 36 insertions(+) diff --git a/packages/opencode/src/altimate/native/finops/databricks-savings.ts b/packages/opencode/src/altimate/native/finops/databricks-savings.ts index d0c2a012ff..d0061befc9 100644 --- a/packages/opencode/src/altimate/native/finops/databricks-savings.ts +++ b/packages/opencode/src/altimate/native/finops/databricks-savings.ts @@ -554,11 +554,18 @@ export async function databricksSavings( clusters.length = 0 jobs.length = 0 } + // Every workspace a checked row came from, whether or not it has a finding or a SQL warehouse. + const workspaces = new Set( + [...warehouses, ...clusters, ...jobs] + .map((row) => (row.workspace_id == null ? "" : String(row.workspace_id).trim())) + .filter((id) => id !== ""), + ) return { success: true, window, warehouse: resolved.warehouse, ran_on: runsOn, + workspaces_checked: workspaces.size, warehouses_checked: warehouses.length, clusters_checked: clusters.length, // The query gives a row per task, cluster and trigger: each job is counted once. diff --git a/packages/opencode/src/altimate/native/types.ts b/packages/opencode/src/altimate/native/types.ts index ea9e52bfe6..4d78d45830 100644 --- a/packages/opencode/src/altimate/native/types.ts +++ b/packages/opencode/src/altimate/native/types.ts @@ -686,6 +686,8 @@ export interface DatabricksSavingsResult { /** The connection that ran the check, and the SQL warehouse it runs on. */ warehouse?: string ran_on?: string + /** Distinct workspaces across every warehouse, cluster and job row checked, with or without a finding. */ + workspaces_checked?: number warehouses_checked?: number clusters_checked?: number jobs_checked?: number diff --git a/packages/opencode/src/altimate/tools/finops-databricks-savings.ts b/packages/opencode/src/altimate/tools/finops-databricks-savings.ts index 480c6df848..867217f35c 100644 --- a/packages/opencode/src/altimate/tools/finops-databricks-savings.ts +++ b/packages/opencode/src/altimate/tools/finops-databricks-savings.ts @@ -10,6 +10,7 @@ type SavingsMetadata = { window?: { start: string; end: string } warehouse?: string ran_on?: string + workspaces_checked?: number warehouses_checked?: number clusters_checked?: number jobs_checked?: number @@ -167,6 +168,7 @@ export const FinopsDatabricksSavingsTool = Tool.define("finops_databricks_saving window: result.window, warehouse: result.warehouse, ran_on: result.ran_on, + workspaces_checked: result.workspaces_checked, warehouses_checked: result.warehouses_checked, clusters_checked: result.clusters_checked, jobs_checked: result.jobs_checked, diff --git a/packages/opencode/test/altimate/finops-databricks-savings.test.ts b/packages/opencode/test/altimate/finops-databricks-savings.test.ts index 493672f1fe..bb39f365fa 100644 --- a/packages/opencode/test/altimate/finops-databricks-savings.test.ts +++ b/packages/opencode/test/altimate/finops-databricks-savings.test.ts @@ -122,6 +122,13 @@ describe("finops_databricks_savings tool", () => { expect(result.metadata.window).toEqual({ start: "2026-11-05", end: "2026-11-12" }) }) + test("the number of workspaces checked reaches the metadata", async () => { + Dispatcher.register("finops.databricks_savings" as any, async () => ({ ...REPORT, workspaces_checked: 3 })) + const tool = await initTool(FinopsDatabricksSavingsTool) + const result = await tool.execute({ confirm: true }, stubCtx()) + expect(result.metadata.workspaces_checked).toBe(3) + }) + test("without the user's go-ahead nothing runs: the agent is told to ask first", async () => { let ran = 0 Dispatcher.register("finops.databricks_savings" as any, async () => { @@ -564,6 +571,24 @@ describe("databricks savings handler over a stubbed connection", () => { ]) }) + test("every workspace a checked row came from is counted, with or without a finding or a SQL warehouse", async () => { + // Workspace 111 has the SQL warehouse; 333 has only a cluster and 222 only a job, and nothing is flagged. + const CLUSTER_ROW = { workspace_id: "333", cluster_id: "c9", cluster_name: "adhoc", auto_termination_minutes: 30, total_node_minutes: 600, active_util_days: 3, total_worker_minutes: 600, idle_node_minutes: 60, total_cost: 12 } + const rows = (...all: Record[]) => ({ columns: Object.keys(all[0]!), rows: all.map((one) => Object.values(one)), truncated: false }) + const spy = connect((sql) => { + if (sql === DATABRICKS_WAREHOUSE_FACTS_SQL) return table(WAREHOUSE_ROW) + if (sql === DATABRICKS_CLUSTER_FACTS_SQL) return rows(CLUSTER_ROW, { ...CLUSTER_ROW, cluster_id: "c10", workspace_id: 333n }) + // A second task of the same workspace, and a row with no workspace, add nothing. + return rows(JOB_ROW, { ...JOB_ROW, task_key: "other" }, { ...JOB_ROW, job_id: "j2", workspace_id: null }) + }) + const result = await databricksSavings({ warehouse: "dbx", now_ms: Date.parse("2026-11-18T10:00:00Z") }, rules) + spy.mockRestore() + expect(result.success).toBe(true) + expect(result.report?.findings).toEqual([]) + expect([result.warehouses_checked, result.clusters_checked, result.jobs_checked]).toEqual([1, 2, 2]) + expect(result.workspaces_checked).toBe(3) + }) + test("connecting counts toward the limit: a connection that never comes up ends the check in time", async () => { // A 30 ms session budget stands in for the last moments of the 9 minutes. const budget = spyOn(RunBudget, "queryDeadline").mockReturnValue(Date.now() + 30) From f59833a4c46213aba6c6cc9469e3d67149d8db9e Mon Sep 17 00:00:00 2001 From: Shreyas Telkar Date: Thu, 19 Nov 2026 16:57:03 -0800 Subject: [PATCH 28/34] docs: assessment mode claims no Altimate contact only for the savings checks The docs and comments said assessment mode keeps any `tool run` from contacting Altimate. Other tools that call the Altimate API directly can still reach it for a signed-in user, so the claim now covers the two savings tools. Co-Authored-By: Claude Opus 5.5 --- docs/docs/data-engineering/tools/finops-tools.md | 2 +- packages/core/src/flag/flag.ts | 2 +- packages/drivers/src/snowflake.ts | 2 +- packages/opencode/src/altimate/api/client.ts | 2 +- packages/opencode/src/altimate/workspace/api-client.ts | 2 +- packages/opencode/src/flag/flag.ts | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/docs/data-engineering/tools/finops-tools.md b/docs/docs/data-engineering/tools/finops-tools.md index 575c4f057c..9aea77ef8c 100644 --- a/docs/docs/data-engineering/tools/finops-tools.md +++ b/docs/docs/data-engineering/tools/finops-tools.md @@ -193,7 +193,7 @@ The tool measures each resource's usage; altimate-core decides what qualifies an **Asks first, and stops long queries.** The check runs only when called with `confirm: true`; without it, the agent is told to ask you first, since a run can start a stopped warehouse that then bills. It runs three queries and stops after 9 minutes, connecting included, cancelling any query still running on the warehouse (or sooner, at an `altimate run --time-budget` cutoff); stopping the session cancels the running query too. The Claude Code `/savings` command passes `confirm: true`, since typing it is the go-ahead. -**Assessment mode.** Set `ALTIMATE_ASSESSMENT_MODE=true` to keep the run from contacting anything but your warehouse. It sends no telemetry (not even the one-time opt-out event), treats you as signed out of Altimate (no workspace or project-binding lookups), fetches no remote config and installs no plugin dependencies, and downloads neither the model catalog (models.dev) nor ripgrep (github.com). Any value but empty, `0` or `false` turns it on, and a tool daemon started without it is not reused. It is measured for `tool run`; other commands (the update check, language-server downloads) are not covered. Core's compiled analysis is not audited for network use. Without it, `ALTIMATE_TELEMETRY_DISABLED=true` alone still lets the CLI send that opt-out event and fetch those. The Claude Code `/savings` command sets it for you. Tool results shown in a Claude Code conversation reach its model provider; assessment mode does not change that. +**Assessment mode.** Set `ALTIMATE_ASSESSMENT_MODE=true` to keep a savings check from contacting anything but your warehouse. It sends no telemetry (not even the one-time opt-out event), treats you as signed out of Altimate (no workspace or project-binding lookups), fetches no remote config and installs no plugin dependencies, and downloads neither the model catalog (models.dev) nor ripgrep (github.com). Any value but empty, `0` or `false` turns it on, and a tool daemon started without it is not reused. It is measured for `tool run` of the two savings tools. Other tools and commands are not covered: a signed-in user's other Altimate tools can still reach Altimate, and so can the update check and language-server downloads. Core's compiled analysis is not audited for network use. Without it, `ALTIMATE_TELEMETRY_DISABLED=true` alone still lets the CLI send that opt-out event and fetch those. The Claude Code `/savings` command sets it for you. Tool results shown in a Claude Code conversation reach its model provider; assessment mode does not change that. ``` > finops_databricks_savings --warehouse prod_dbx --confirm diff --git a/packages/core/src/flag/flag.ts b/packages/core/src/flag/flag.ts index 624e3687a5..a9d2df8ecc 100644 --- a/packages/core/src/flag/flag.ts +++ b/packages/core/src/flag/flag.ts @@ -97,7 +97,7 @@ export const Flag = { OPENCODE_DISABLE_TERMINAL_TITLE: truthy("OPENCODE_DISABLE_TERMINAL_TITLE"), OPENCODE_SHOW_TTFD: truthy("OPENCODE_SHOW_TTFD"), OPENCODE_DISABLE_AUTOCOMPACT: truthy("OPENCODE_DISABLE_AUTOCOMPACT"), - // altimate_change start — assessment mode: nothing leaves the machine except the customer's own warehouse calls + // altimate_change start — assessment mode: a savings check contacts nothing but the customer's own warehouse // Fails closed like the other kill switches: `yes` or `on` must not leave the CLI contacting hosts. get ALTIMATE_ASSESSMENT_MODE() { return killSwitchSet(process.env["ALTIMATE_ASSESSMENT_MODE"]) diff --git a/packages/drivers/src/snowflake.ts b/packages/drivers/src/snowflake.ts index 52334c2314..d54605f3b9 100644 --- a/packages/drivers/src/snowflake.ts +++ b/packages/drivers/src/snowflake.ts @@ -57,7 +57,7 @@ export function suppressSnowflakeLogging(snowflake: any): void { } /** - * In assessment mode nothing but the warehouse may be contacted. When it loads, snowflake-sdk otherwise probes + * In assessment mode a savings check may contact nothing but the warehouse. When it loads, snowflake-sdk otherwise probes * the AWS, Azure and GCP metadata and identity services to report the platform it runs on. Its certificate * revocation requests need no switch here: they start from an HTTPS agent that Bun, which runs the packaged * CLI, does not use. diff --git a/packages/opencode/src/altimate/api/client.ts b/packages/opencode/src/altimate/api/client.ts index 943cf54a90..ef31dda3f0 100644 --- a/packages/opencode/src/altimate/api/client.ts +++ b/packages/opencode/src/altimate/api/client.ts @@ -89,7 +89,7 @@ export namespace AltimateApi { } export async function isConfigured(): Promise { - // Assessment mode talks to no Altimate service: to every caller the user is signed out. + // In assessment mode every caller that checks this treats the user as signed out. if (Flag.assessmentMode()) return false return credentialsPresent() } diff --git a/packages/opencode/src/altimate/workspace/api-client.ts b/packages/opencode/src/altimate/workspace/api-client.ts index 862d0c04cd..46c6640565 100644 --- a/packages/opencode/src/altimate/workspace/api-client.ts +++ b/packages/opencode/src/altimate/workspace/api-client.ts @@ -430,7 +430,7 @@ async function req( // altimate_change end } = {}, ): Promise { - // altimate_change start — assessment mode talks to no Altimate service, whoever the caller acts as + // altimate_change start — in assessment mode the workspace client contacts no Altimate service, whoever the caller acts as if (Flag.assessmentMode()) throw new NotConfiguredError() // altimate_change end const timeoutMs = opts.timeoutMs ?? REQUEST_TIMEOUT_MS diff --git a/packages/opencode/src/flag/flag.ts b/packages/opencode/src/flag/flag.ts index f01cb4d845..9b343b9938 100644 --- a/packages/opencode/src/flag/flag.ts +++ b/packages/opencode/src/flag/flag.ts @@ -113,7 +113,7 @@ export namespace Flag { export const OPENCODE_DISABLE_LSP_DOWNLOAD = truthy("OPENCODE_DISABLE_LSP_DOWNLOAD") export const OPENCODE_ENABLE_EXPERIMENTAL_MODELS = truthy("OPENCODE_ENABLE_EXPERIMENTAL_MODELS") export const OPENCODE_DISABLE_AUTOCOMPACT = truthy("OPENCODE_DISABLE_AUTOCOMPACT") - // altimate_change start — assessment mode: nothing leaves the machine except the customer's own warehouse calls + // altimate_change start — assessment mode: a savings check contacts nothing but the customer's own warehouse /** Read live; fails closed (any value but empty, `0` or `false` turns it on). */ export function assessmentMode(): boolean { return killSwitchSet(process.env["ALTIMATE_ASSESSMENT_MODE"]) From 7400c9b6e9e7cdc351026a57f590c9aaf8a96b59 Mon Sep 17 00:00:00 2001 From: Shreyas Telkar Date: Thu, 19 Nov 2026 17:05:28 -0800 Subject: [PATCH 29/34] fix: a Snowflake query with no cloud services credits keeps its compute share A NULL `CREDITS_USED_CLOUD_SERVICES` made the short-query credit sum skip the whole query, compute included. It now counts as zero cloud credits. Co-Authored-By: Claude Opus 5.5 --- .../opencode/src/altimate/native/finops/snowflake-savings.ts | 2 +- .../opencode/test/altimate/finops-snowflake-savings.test.ts | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/opencode/src/altimate/native/finops/snowflake-savings.ts b/packages/opencode/src/altimate/native/finops/snowflake-savings.ts index fb0d3e0739..ed6fc2d4bc 100644 --- a/packages/opencode/src/altimate/native/finops/snowflake-savings.ts +++ b/packages/opencode/src/altimate/native/finops/snowflake-savings.ts @@ -218,7 +218,7 @@ SELECT q.warehouse_id, q.warehouse_name, COUNT_IF(q.execution_ms <= 5000) AS sub_five_second_queries, COALESCE(SUM(IFF(q.execution_ms <= 5000, COALESCE(a.compute_credits, 0) - + IFF(d.cloud_credits > 0, q.cloud_credits / d.cloud_credits * d.billable_cloud_credits, 0), 0)), 0) AS sub_five_second_credits + + IFF(d.cloud_credits > 0, COALESCE(q.cloud_credits, 0) / d.cloud_credits * d.billable_cloud_credits, 0), 0)), 0) AS sub_five_second_credits FROM q LEFT JOIN allocated a ON a.query_id = q.query_id LEFT JOIN daily d ON d.day = q.st::DATE diff --git a/packages/opencode/test/altimate/finops-snowflake-savings.test.ts b/packages/opencode/test/altimate/finops-snowflake-savings.test.ts index 78ef11bf21..22674f5954 100644 --- a/packages/opencode/test/altimate/finops-snowflake-savings.test.ts +++ b/packages/opencode/test/altimate/finops-snowflake-savings.test.ts @@ -216,6 +216,8 @@ describe("the fact queries", () => { expect(SNOWFLAKE_PERIOD_FACTS_SQL).toContain("DATEDIFF('second', valid_from, valid_to)") expect(SNOWFLAKE_QUERY_FACTS_SQL).toContain("COUNT_IF(q.execution_ms <= 5000)") expect(SNOWFLAKE_QUERY_FACTS_SQL).toContain("SUM(compute_credits) * 0.1") + // A query with no cloud services credits still counts its share of compute credits. + expect(SNOWFLAKE_QUERY_FACTS_SQL).toContain("COALESCE(q.cloud_credits, 0) / d.cloud_credits") }) test("the windows and filters are the ones the rules are defined on", () => { From 211c191ef979046c47321fad81764fea8955f153 Mon Sep 17 00:00:00 2001 From: Shreyas Telkar Date: Thu, 19 Nov 2026 17:44:11 -0800 Subject: [PATCH 30/34] fix: Databricks and Snowflake cancel an aborted statement without a deadline - An aborted `signal` now cancels the statement on the warehouse even when the caller set no `cancelAfterMs`, as `ExecuteOptions.signal` documents. - Snowflake: a statement stopped while still waiting on a reconnect is reported as never sent (`unsent`), not as cancelled. - Databricks: a failed close after the rows were fetched no longer fails the query; one `closeQuietly` helper covers every close. - Both drivers wait through one shared `untilStopped` helper. Co-Authored-By: Claude Opus 5.5 --- packages/drivers/src/databricks.ts | 58 +++++++---------- packages/drivers/src/snowflake.ts | 41 ++++-------- packages/drivers/src/stop.ts | 46 ++++++++++++++ packages/drivers/src/types.ts | 20 ++++-- .../drivers/test/databricks-deadline.test.ts | 40 +++++++++++- .../drivers/test/snowflake-deadline.test.ts | 28 ++++++++- packages/drivers/test/stop.test.ts | 63 +++++++++++++++++++ 7 files changed, 221 insertions(+), 75 deletions(-) create mode 100644 packages/drivers/src/stop.ts create mode 100644 packages/drivers/test/stop.test.ts diff --git a/packages/drivers/src/databricks.ts b/packages/drivers/src/databricks.ts index 306e6f4549..708211fb7b 100644 --- a/packages/drivers/src/databricks.ts +++ b/packages/drivers/src/databricks.ts @@ -4,7 +4,7 @@ import type { ConnectionConfig, Connector, ConnectorResult, ExecuteOptions, SchemaColumn } from "./types" import { QueryTimeoutError } from "./types" -import { MAX_TIMER_MS } from "./query-timeout" +import { untilStopped, type Stopped } from "./stop" import { loadOptionalDriver } from "./resolve" export async function connect(config: ConnectionConfig): Promise { @@ -22,7 +22,9 @@ export async function connect(config: ConnectionConfig): Promise { // How long to wait for the server to acknowledge a cancel before reporting the statement as // possibly still running. const CANCEL_WAIT_MS = 30_000 - const STOPPED = Symbol("stopped") + + /** Frees the statement on the server. A failed close never replaces the query's own outcome. */ + const closeQuietly = (operation: any): Promise => operation.close().catch(() => {}) async function cancelAndClose(operation: any): Promise { let timer: ReturnType | undefined @@ -35,19 +37,20 @@ export async function connect(config: ConnectionConfig): Promise { timer = setTimeout(() => resolve(false), CANCEL_WAIT_MS) }), ]).finally(() => clearTimeout(timer)) - operation.close().catch(() => {}) + void closeQuietly(operation) return cancelled } - // `cancelAfterMs` is an opt-in deadline: past it, or when `signal` aborts, the statement is - // cancelled on the warehouse. SQL warehouses ignore the client's server-side `queryTimeout`. + // Past `cancelAfterMs`, or when `signal` aborts, the statement is cancelled on the warehouse: SQL + // warehouses ignore the client's server-side `queryTimeout`. With neither, it runs to the end. async function runStatement( query: string, stmtOptions: Record, options?: ExecuteOptions, ): Promise { - const deadlineMs = options?.cancelAfterMs - if (!deadlineMs || deadlineMs <= 0) { + const deadlineMs = options?.cancelAfterMs && options.cancelAfterMs > 0 ? options.cancelAfterMs : undefined + const signal = options?.signal + if (deadlineMs === undefined && !signal) { const operation = await session.executeStatement(query, stmtOptions) let rows: any[] try { @@ -55,29 +58,14 @@ export async function connect(config: ConnectionConfig): Promise { } catch (error) { // A statement that failed while fetching is still open on the session: free it, and // report the fetch's own error rather than any from the close. - operation.close().catch(() => {}) + void closeQuietly(operation) throw error } - await operation.close() + await closeQuietly(operation) return rows } - const signal = options?.signal signal?.throwIfAborted() - let stopped: "deadline" | "aborted" | undefined - let stop: () => void = () => {} - const stopping = new Promise((resolve) => (stop = () => resolve(STOPPED))) - const timer = setTimeout( - () => { - stopped ??= "deadline" - stop() - }, - Math.min(deadlineMs, MAX_TIMER_MS), - ) - const onAbort = () => { - stopped ??= "aborted" - stop() - } - signal?.addEventListener("abort", onAbort, { once: true }) + let stopped = false let operation: any const work = (async () => { const started = await session.executeStatement(query, stmtOptions) @@ -89,27 +77,23 @@ export async function connect(config: ConnectionConfig): Promise { operation = started return operation.fetchAll() as Promise })() - let outcome: any[] | typeof STOPPED + let outcome: Stopped try { - outcome = await Promise.race([work, stopping]) + outcome = await untilStopped(work, { deadlineMs, signal, onStop: () => (stopped = true) }) } catch (error) { // The fetch failed before any stop: the statement is still open on the session. - operation?.close().catch(() => {}) + if (operation) void closeQuietly(operation) throw error - } finally { - clearTimeout(timer) - signal?.removeEventListener("abort", onAbort) } - if (outcome !== STOPPED) { + if (!outcome.stopped) { // Closing frees the server's result; a slow or failed close must not hold up or fail a query // that already returned. - operation.close().catch(() => {}) - return outcome + void closeQuietly(operation) + return outcome.value } - work.catch(() => {}) const cancelled = operation ? await cancelAndClose(operation) : false - if (stopped === "aborted") throw signal!.reason - throw new QueryTimeoutError({ engine: "databricks", timeoutMs: deadlineMs, cancelled }) + if (outcome.stopped === "aborted") throw signal!.reason + throw new QueryTimeoutError({ engine: "databricks", timeoutMs: deadlineMs!, cancelled }) } return { diff --git a/packages/drivers/src/snowflake.ts b/packages/drivers/src/snowflake.ts index d54605f3b9..efc7302e58 100644 --- a/packages/drivers/src/snowflake.ts +++ b/packages/drivers/src/snowflake.ts @@ -6,7 +6,7 @@ import * as fs from "fs" import type { ConnectionConfig, Connector, ConnectorResult, ExecuteOptions, SchemaColumn } from "./types" import { toPkcs8Pem } from "./snowflake-key" import { QueryTimeoutError } from "./types" -import { MAX_TIMER_MS } from "./query-timeout" +import { untilStopped } from "./stop" import { loadOptionalDriver } from "./resolve" import { emitBrowserSignIn, openInBrowser } from "./sign-in" import { emitReconnect, type ReconnectEvent } from "./reconnect-events" @@ -597,40 +597,25 @@ export async function connect( } /** - * `cancelAfterMs` is an opt-in deadline: past it, or when `signal` aborts, the statement is cancelled on - * Snowflake. One still waiting on a reconnect is never sent, so nothing is left running. + * Past `cancelAfterMs`, or when `signal` aborts, the statement is cancelled on Snowflake. One still waiting on + * a reconnect is never sent, so nothing is left running. */ async function executeWithCancel( sql: string, binds: any[] | undefined, - deadlineMs: number, + deadlineMs: number | undefined, signal?: AbortSignal, ): Promise<{ columns: string[]; rows: any[][] }> { signal?.throwIfAborted() const stop: StatementStop = { statements: [] } - const STOPPED = Symbol("stopped") - let wake: () => void = () => {} - const stopping = new Promise((resolve) => (wake = () => resolve(STOPPED))) - const halt = (why: "deadline" | "aborted") => { - stop.stopped ??= why - wake() - } - const timer = setTimeout(() => halt("deadline"), Math.min(deadlineMs, MAX_TIMER_MS)) - const onAbort = () => halt("aborted") - signal?.addEventListener("abort", onAbort, { once: true }) const work = executeQuery(sql, binds, stop) - let outcome: { columns: string[]; rows: any[][] } | typeof STOPPED - try { - outcome = await Promise.race([work, stopping]) - } finally { - clearTimeout(timer) - signal?.removeEventListener("abort", onAbort) - } - if (outcome !== STOPPED) return outcome - work.catch(() => {}) - const cancelled = (await Promise.all(stop.statements.map(cancelStatement))).every(Boolean) - if (stop.stopped === "aborted") throw signal!.reason - throw new QueryTimeoutError({ engine: "snowflake", timeoutMs: deadlineMs, cancelled }) + const outcome = await untilStopped(work, { deadlineMs, signal, onStop: (why) => (stop.stopped = why) }) + if (!outcome.stopped) return outcome.value + // A statement still waiting on a reconnect was never sent: nothing ran, so nothing was cancelled. + const unsent = stop.statements.length === 0 + const cancelled = !unsent && (await Promise.all(stop.statements.map(cancelStatement))).every(Boolean) + if (outcome.stopped === "aborted") throw signal!.reason + throw new QueryTimeoutError({ engine: "snowflake", timeoutMs: deadlineMs!, cancelled, unsent }) } return { @@ -798,9 +783,9 @@ export async function connect( query = `${sql.replace(/;\s*$/, "")} LIMIT ${effectiveLimit + 1}` } - const deadlineMs = options?.cancelAfterMs + const deadlineMs = options?.cancelAfterMs && options.cancelAfterMs > 0 ? options.cancelAfterMs : undefined const result = - deadlineMs && deadlineMs > 0 + deadlineMs !== undefined || options?.signal ? await executeWithCancel(query, binds, deadlineMs, options?.signal) : await executeQuery(query, binds) const truncated = effectiveLimit > 0 && result.rows.length > effectiveLimit diff --git a/packages/drivers/src/stop.ts b/packages/drivers/src/stop.ts new file mode 100644 index 0000000000..c5913336d7 --- /dev/null +++ b/packages/drivers/src/stop.ts @@ -0,0 +1,46 @@ +/** + * Waiting on work that a deadline or an abort signal may cut short. The drivers use it to cancel a + * statement on the server once it is stopped; the savings checks use it to stop waiting on a connect. + */ + +import { MAX_TIMER_MS } from "./query-timeout" + +export type StopReason = "deadline" | "aborted" + +export type Stopped = { stopped?: undefined; value: T } | { stopped: StopReason } + +/** + * Waits for `work`, or until `deadlineMs` passes or `signal` aborts, whichever comes first. `onStop` runs + * at the moment of the stop, before this returns, so work still in flight can see it. A stopped + * `work` that fails later does not go unhandled. No deadline waits on the signal alone. + */ +export async function untilStopped( + work: Promise, + opts: { deadlineMs?: number; signal?: AbortSignal; onStop?: (why: StopReason) => void }, +): Promise> { + const { deadlineMs, signal } = opts + let stopped: StopReason | undefined + let wake: () => void = () => {} + const stopping = new Promise((resolve) => (wake = resolve)) + const halt = (why: StopReason) => { + if (stopped) return + stopped = why + opts.onStop?.(why) + wake() + } + const timer = deadlineMs === undefined ? undefined : setTimeout(() => halt("deadline"), Math.min(Math.max(0, deadlineMs), MAX_TIMER_MS)) + const onAbort = () => halt("aborted") + if (signal?.aborted) onAbort() + else signal?.addEventListener("abort", onAbort, { once: true }) + try { + const value = await Promise.race([work, stopping]) + if (stopped) { + work.catch(() => {}) + return { stopped } + } + return { value: value as T } + } finally { + clearTimeout(timer) + signal?.removeEventListener("abort", onAbort) + } +} diff --git a/packages/drivers/src/types.ts b/packages/drivers/src/types.ts index e3046e0057..e50c0c3f58 100644 --- a/packages/drivers/src/types.ts +++ b/packages/drivers/src/types.ts @@ -44,9 +44,10 @@ export interface ExecuteOptions { * `query_timeout_ms`, then `ALTIMATE_SQL_QUERY_TIMEOUT_MS`, then the * 120 s default (see `query-timeout.ts`). */ queryTimeoutMs?: number - /** Cancel the statement on the server once this many milliseconds pass, or when - * `signal` aborts. Opt-in: `executeQuery` fills in `queryTimeoutMs` for every - * call, so only callers that set this get the server-side cancel. Databricks and Snowflake. */ + /** Cancel the statement on the server once this many milliseconds pass. Opt-in: + * `executeQuery` fills in `queryTimeoutMs` for every call, which cancels nothing, so + * only callers that set this get a server-side deadline. An aborted `signal` cancels + * the statement with or without it. Databricks and Snowflake. */ cancelAfterMs?: number /** DuckDB only: called once after handing this query to its helper; callback errors do not affect execution. */ onDispatch?: () => void @@ -69,6 +70,8 @@ export class QueryTimeoutError extends Error { readonly cancelled: boolean /** Session state discarded to stop the query (temp tables, `:memory:` data). */ readonly stateLost: boolean + /** Stopped before the statement was sent: nothing ran, so nothing needed cancelling. */ + readonly unsent: boolean constructor(opts: { engine: string @@ -76,11 +79,17 @@ export class QueryTimeoutError extends Error { cancelled?: boolean stateLost?: boolean detail?: string + unsent?: boolean }) { - const cancelled = opts.cancelled ?? true + const unsent = opts.unsent ?? false + const cancelled = !unsent && (opts.cancelled ?? true) super( `${QUERY_TIMEOUT_CODE}: ${opts.engine} query exceeded the ${formatDeadline(opts.timeoutMs)} execution deadline ` + - (cancelled ? "and was cancelled. " : "and was abandoned; it may still be running on the server. ") + + (unsent + ? "before it was sent, so it did not run. " + : cancelled + ? "and was cancelled. " + : "and was abandoned; it may still be running on the server. ") + (opts.detail ? `${opts.detail} ` : "") + "Narrow the query (filters, a smaller sample, pre-aggregation) or set query_timeout_ms on the connection " + "or ALTIMATE_SQL_QUERY_TIMEOUT_MS to allow longer runs.", @@ -90,6 +99,7 @@ export class QueryTimeoutError extends Error { this.engine = opts.engine this.cancelled = cancelled this.stateLost = opts.stateLost ?? false + this.unsent = unsent } } diff --git a/packages/drivers/test/databricks-deadline.test.ts b/packages/drivers/test/databricks-deadline.test.ts index 4f262da714..a0b1c86473 100644 --- a/packages/drivers/test/databricks-deadline.test.ts +++ b/packages/drivers/test/databricks-deadline.test.ts @@ -1,7 +1,7 @@ /** * Databricks server-side cancel: a caller that sets `cancelAfterMs` gets its statement cancelled - * on the warehouse when the deadline passes or its signal aborts. `queryTimeoutMs`, which every - * `executeQuery` call fills in, does not cancel anything. + * on the warehouse when the deadline passes, and an aborted signal cancels it with or without one. + * `queryTimeoutMs`, which every `executeQuery` call fills in, does not cancel anything. */ import { beforeEach, describe, expect, mock, test } from "bun:test" import { restoreModuleMocksAfterTests } from "./module-mock" @@ -182,4 +182,40 @@ describe("Databricks statement cleanup when the fetch fails", () => { expect(result.rows).toEqual([[1]]) expect([cancels, closes]).toEqual([0, 1]) }) + + test("a close that fails after the rows were fetched does not fail the query", async () => { + closeResult = async () => { + throw new Error("close failed") + } + const db = await connector() + const result = await db.execute("SELECT 1", 10, []) + expect(result.rows).toEqual([[1]]) + expect(closes).toBe(1) + }) +}) + +describe("Databricks cancel on abort without a deadline", () => { + test("without `cancelAfterMs`, an aborted signal still cancels the statement on the warehouse", async () => { + fetch = never + const controller = new AbortController() + const db = await connector() + const pending = db.execute("SELECT 1", 10, [], { queryTimeoutMs: 60_000, signal: controller.signal }).catch((e) => e) + await sleep(20) + controller.abort(new Error("session stopped")) + const error = await pending + expect(isQueryTimeoutError(error)).toBe(false) + expect(error.message).toBe("session stopped") + expect([cancels, closes]).toEqual([1, 1]) + }) + + test("with a signal that never aborts and no `cancelAfterMs`, a slow statement still returns its rows", async () => { + fetch = async () => { + await sleep(40) + return [{ n: 1 }] + } + const db = await connector() + const result = await db.execute("SELECT 1", 10, [], { signal: new AbortController().signal }) + expect(result.rows).toEqual([[1]]) + expect(cancels).toBe(0) + }) }) diff --git a/packages/drivers/test/snowflake-deadline.test.ts b/packages/drivers/test/snowflake-deadline.test.ts index ba1a3d25b2..c807cc9d5b 100644 --- a/packages/drivers/test/snowflake-deadline.test.ts +++ b/packages/drivers/test/snowflake-deadline.test.ts @@ -1,6 +1,6 @@ /** * Snowflake server-side cancel: a caller-named `cancelAfterMs` cancels the statement when it runs - * out or when `signal` aborts; without one, execution is unchanged. + * out, and an aborted `signal` cancels it with or without one; with neither, execution is unchanged. */ import { beforeEach, describe, expect, test } from "bun:test" import { connect, keepSdkOffCloudMetadata } from "../src/snowflake" @@ -135,6 +135,25 @@ describe("Snowflake server-side cancel", () => { expect(cancels).toBe(1) }) + test("without `cancelAfterMs`, an aborted signal still cancels the statement on Snowflake", async () => { + const controller = new AbortController() + const db = await connector() + const pending = db.execute("SELECT 1", 10, [], { queryTimeoutMs: 60_000, signal: controller.signal }).catch((e) => e) + await sleep(10) + controller.abort(new Error("session stopped")) + const error = await pending + expect(error.message).toBe("session stopped") + expect(cancels).toBe(1) + }) + + test("with a signal that never aborts and no `cancelAfterMs`, a slow statement still returns its rows", async () => { + finishAfterMs = 40 + const db = await connector() + const result = await db.execute("SELECT 1", 10, [], { signal: new AbortController().signal }) + expect(result.rows).toEqual([[1]]) + expect(cancels).toBe(0) + }) + test("an already-aborted signal sends nothing", async () => { const controller = new AbortController() controller.abort(new Error("session stopped")) @@ -151,8 +170,11 @@ describe("Snowflake server-side cancel", () => { up = false const error = await db.execute("SELECT 1", 10, [], { cancelAfterMs: 30 }).catch((e) => e) expect(isQueryTimeoutError(error)).toBe(true) - // Nothing reached Snowflake, so nothing is left running there. - expect(error.cancelled).toBe(true) + // Nothing reached Snowflake: nothing ran, so nothing was cancelled, and the error says so. + expect(error.unsent).toBe(true) + expect(error.cancelled).toBe(false) + expect(error.message).toContain("before it was sent, so it did not run") + expect(error.message).not.toContain("may still be running") expect(cancels).toBe(0) up = true release() diff --git a/packages/drivers/test/stop.test.ts b/packages/drivers/test/stop.test.ts new file mode 100644 index 0000000000..c29ff19515 --- /dev/null +++ b/packages/drivers/test/stop.test.ts @@ -0,0 +1,63 @@ +import { describe, expect, test } from "bun:test" +import { untilStopped } from "../src/stop" + +const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)) +const never = () => new Promise(() => {}) + +describe("untilStopped", () => { + test("work that settles first is returned and nothing is stopped", async () => { + const stops: string[] = [] + const outcome = await untilStopped(Promise.resolve(7), { deadlineMs: 1_000, onStop: (why) => stops.push(why) }) + expect(outcome).toEqual({ value: 7 }) + expect(stops).toEqual([]) + }) + + test("a deadline stops the wait and calls onStop once", async () => { + const stops: string[] = [] + const outcome = await untilStopped(never(), { deadlineMs: 10, onStop: (why) => stops.push(why) }) + expect(outcome).toEqual({ stopped: "deadline" }) + expect(stops).toEqual(["deadline"]) + }) + + test("an abort stops the wait, with no deadline at all", async () => { + const controller = new AbortController() + const pending = untilStopped(never(), { signal: controller.signal }) + await sleep(5) + controller.abort(new Error("stop")) + expect(await pending).toEqual({ stopped: "aborted" }) + }) + + test("an already-aborted signal stops at once", async () => { + const controller = new AbortController() + controller.abort(new Error("stop")) + expect(await untilStopped(never(), { deadlineMs: 1_000, signal: controller.signal })).toEqual({ stopped: "aborted" }) + }) + + test("work that fails after the stop does not go unhandled", async () => { + const unhandled: unknown[] = [] + const onUnhandled = (error: unknown) => unhandled.push(error) + process.on("unhandledRejection", onUnhandled) + try { + const late = sleep(20).then(() => Promise.reject(new Error("late"))) + expect(await untilStopped(late, { deadlineMs: 5 })).toEqual({ stopped: "deadline" }) + await sleep(40) + expect(unhandled).toEqual([]) + } finally { + process.off("unhandledRejection", onUnhandled) + } + }) + + test("work that fails before any stop rejects with its own error and leaves no listener", async () => { + const controller = new AbortController() + let listeners = 0 + const add = controller.signal.addEventListener.bind(controller.signal) + const remove = controller.signal.removeEventListener.bind(controller.signal) + controller.signal.addEventListener = ((type: string, fn: never, options: never) => (listeners++, add(type, fn, options))) as never + controller.signal.removeEventListener = ((type: string, fn: never) => (listeners--, remove(type, fn))) as never + const error = await untilStopped(Promise.reject(new Error("query failed")), { deadlineMs: 1_000, signal: controller.signal }).catch( + (e) => e, + ) + expect(error.message).toBe("query failed") + expect(listeners).toBe(0) + }) +}) From f5ef7a1cfae5af80a47777ee85fbde8e8d81168b Mon Sep 17 00:00:00 2001 From: Shreyas Telkar Date: Thu, 19 Nov 2026 18:01:22 -0800 Subject: [PATCH 31/34] fix: the savings checks start no work after a stop and read the rate sheet last - Both checks share one run helper (`savings-run.ts`): connecting starts only while the session runs and time is left, so a stopped or out-of-budget run no longer opens a warehouse connection or sign-in. - Snowflake reads the optional rate sheet after the usage queries, so a slow rate lookup can no longer leave a required query without time. - Snowflake names usage an earlier warehouse billed under a reused name as not checked, instead of dropping it silently. - A query stopped before it was sent says it did not run. - A hand-edited connection whose `type` is not text is passed over. - Shared formatters live in `finops-formatting.ts`; the 9-minute text comes from the limit; the list-price join and core's telemetry switch have one definition each. - Tests restore their spies after each test; the live job check asserts no check was skipped; the docs show the what-if block and how idle time and shared-cluster jobs are counted. Co-Authored-By: Claude Opus 5.5 --- .../data-engineering/tools/finops-tools.md | 11 +- .../src/altimate/native/core-telemetry.ts | 14 +- .../native/finops/databricks-savings.ts | 89 +++----- .../src/altimate/native/finops/savings-run.ts | 70 +++++++ .../native/finops/snowflake-savings.ts | 98 ++++----- .../tools/finops-databricks-savings.ts | 29 +-- .../src/altimate/tools/finops-formatting.ts | 17 ++ .../tools/finops-snowflake-savings.ts | 7 +- .../altimate/finops-databricks-e2e.test.ts | 5 + .../finops-databricks-savings.test.ts | 83 +++++--- .../altimate/finops-snowflake-savings.test.ts | 191 ++++++++++++------ 11 files changed, 357 insertions(+), 257 deletions(-) create mode 100644 packages/opencode/src/altimate/native/finops/savings-run.ts diff --git a/docs/docs/data-engineering/tools/finops-tools.md b/docs/docs/data-engineering/tools/finops-tools.md index 9aea77ef8c..302c462a6f 100644 --- a/docs/docs/data-engineering/tools/finops-tools.md +++ b/docs/docs/data-engineering/tools/finops-tools.md @@ -189,7 +189,7 @@ Check a Databricks workspace for savings with Altimate's rules, on a Databricks The tool measures each resource's usage; altimate-core decides what qualifies and what it saves. Prices are list prices. Each resource is counted once in the total, and each flagged warehouse also gets what it would cost at every smaller size and as serverless. If the connection cannot read the cluster or job tables, that check is named as not checked and the warehouse checks still report; the warehouse tables themselves are required. -**What it does not assess.** Databricks Apps spend is not covered, and a serverless SQL warehouse gets only the Over-Sized Warehouse check. A SQL warehouse's cost is its billed usage for the six full days before the window's last day, counting only records Databricks had ingested before that day, spread over the seven-day window as Altimate's pipeline does, so it reads lower than seven days of billing (on one measured workspace, about 7% lower). A cluster's cost is its billed usage for the seven full days before the window's last day. A job's cost is estimated, because Databricks billing does not name the job on all-purpose compute: its task hours from the job timeline times the DBUs per hour its cluster billed. Jobs sharing a cluster never add up to more DBUs than the cluster billed. The connection needs the `@databricks/sql` driver installed. +**What it does not assess.** Databricks Apps spend is not covered, and a serverless SQL warehouse gets only the Over-Sized Warehouse check. A SQL warehouse's cost is its billed usage for the six full days before the window's last day, counting only records Databricks had ingested before that day, spread over the seven-day window as Altimate's pipeline does, so it reads lower than seven days of billing (on one measured workspace, about 7% lower). A cluster's cost is its billed usage for the seven full days before the window's last day. A job's cost is estimated, because Databricks billing does not name the job on all-purpose compute: its task hours from the job timeline times the DBUs per hour its cluster billed. When the jobs on one cluster would add up to more DBUs than it billed, their estimates are scaled down to fit. The connection needs the `@databricks/sql` driver installed. **Asks first, and stops long queries.** The check runs only when called with `confirm: true`; without it, the agent is told to ask you first, since a run can start a stopped warehouse that then bills. It runs three queries and stops after 9 minutes, connecting included, cancelling any query still running on the warehouse (or sooner, at an `altimate run --time-budget` cutoff); stopping the session cancels the running query too. The Claude Code `/savings` command passes `confirm: true`, since typing it is the go-ahead. @@ -204,6 +204,11 @@ Total: $85.17–$106 a month on 1 resource, each counted once at its largest sav Classic (sql warehouse, workspace 525779153077873) · Migrate Classic/Pro to Serverless · $85.17–$106 a month (critical) CLASSIC SMALL warehouse, 95% idle over 10.8 running hours (2,988 queries) $29.29 in DBUs over the window; serverless would bill $4.44 for the busy time alone + +What Classic (now SMALL (CLASSIC), $125 a month) would cost instead: + SERVERLESS: $19.03 a month, saving $106 + X-SMALL: $62.60 a month, saving $62.60 + 2X-SMALL: $41.73 a month, saving $83.47 ``` @@ -211,7 +216,7 @@ Classic (sql warehouse, workspace 525779153077873) · Migrate Classic/Pro to Ser ## finops_snowflake_savings -Check a Snowflake account for savings with Altimate's rules. It lists warehouses with `SHOW WAREHOUSES` and, to report on seven days, reads up to eleven days of `SNOWFLAKE.ACCOUNT_USAGE` (`WAREHOUSE_METERING_HISTORY`, `WAREHOUSE_EVENTS_HISTORY`, `QUERY_HISTORY`), read-only, on the connection's warehouse. The role needs `IMPORTED PRIVILEGES` on the `SNOWFLAKE` database, and sees only the warehouses it has access to; warehouses that billed credits but cannot be listed are named as not checked. +Check a Snowflake account for savings with Altimate's rules. It lists warehouses with `SHOW WAREHOUSES` and, to report on seven days, reads up to eleven days of `SNOWFLAKE.ACCOUNT_USAGE` (`WAREHOUSE_METERING_HISTORY`, `WAREHOUSE_EVENTS_HISTORY`, `QUERY_HISTORY`), read-only, on the connection's warehouse. The role needs `IMPORTED PRIVILEGES` on the `SNOWFLAKE` database, and sees only the warehouses it has access to; warehouses that billed credits but cannot be listed are named as not checked, and so is usage an earlier warehouse billed under a name a newer one now has. - **High Idle Time:** on days a warehouse sat idle for more than 40% of its active time, the idle share of that day's cost. - **Downsize Warehouse:** nearly all queries finish within 5 seconds, or nearly all active periods last 80 seconds or less, so one size smaller would do. @@ -220,7 +225,7 @@ Check a Snowflake account for savings with Altimate's rules. It lists warehouses The tool measures each warehouse's usage; altimate-core decides what qualifies and what it saves. Credits are priced from `ORGANIZATION_USAGE.RATE_SHEET_DAILY` when the role can read it in dollars, otherwise at $3.00 (also for an account billed in another currency), and the output says which. Each warehouse is counted once in the total. -**What it does not assess.** These are four of Altimate's warehouse checks. Storage, serverless features and individual query tuning are not covered. A warehouse that never suspended in the window has no completed active period, so it gets no idle finding. Days are UTC, and the window ends at 00:00 UTC on the latest day at least six hours past, because `ACCOUNT_USAGE` lags by up to that long. +**What it does not assess.** These are four of Altimate's warehouse checks. Storage, serverless features and individual query tuning are not covered. As in Altimate's hosted product, idle time is measured within active periods (resume to the next suspend or resume) that ran at least one query, and a period counts on the day it started: a warehouse that never suspended in the window has no completed active period, and one resumed and left running with no query at all adds no idle time, so neither gets an idle finding for that time. Days are UTC, and the window ends at 00:00 UTC on the latest day at least six hours past, because `ACCOUNT_USAGE` lags by up to that long. **Asks first, and stops long queries.** The check runs only when called with `confirm: true`; without it, the agent is told to ask you first, since a run can resume a suspended warehouse that then bills. It runs six queries and stops after 9 minutes, connecting included, cancelling any query still running on Snowflake (or sooner, at an `altimate run --time-budget` cutoff); stopping the session cancels the running query too. diff --git a/packages/opencode/src/altimate/native/core-telemetry.ts b/packages/opencode/src/altimate/native/core-telemetry.ts index f49beca453..3d71cb59db 100644 --- a/packages/opencode/src/altimate/native/core-telemetry.ts +++ b/packages/opencode/src/altimate/native/core-telemetry.ts @@ -4,12 +4,18 @@ * this process does not reach it, so assessment mode switches it off through `initSdk`, which only * stores the setting. A core that cannot load has nothing to send. */ +export type CoreTelemetrySwitch = { + initSdk?: (apiKey: string, tenant: string, backendUrl: string, reportFailures: boolean, telemetry: boolean) => void +} + +/** Switches the usage event off on a core already loaded. */ +export function silenceWith(core: CoreTelemetrySwitch): void { + core.initSdk?.("", "", "", false, false) +} + export async function silence(): Promise { try { - const core = (await import("@altimateai/altimate-core")) as unknown as { - initSdk?: (apiKey: string, tenant: string, backendUrl: string, reportFailures: boolean, telemetry: boolean) => void - } - core.initSdk?.("", "", "", false, false) + silenceWith((await import("@altimateai/altimate-core")) as unknown as CoreTelemetrySwitch) } catch { // No core binary on this platform: no core call can run, so none can send. } diff --git a/packages/opencode/src/altimate/native/finops/databricks-savings.ts b/packages/opencode/src/altimate/native/finops/databricks-savings.ts index d0061befc9..c60f5c991c 100644 --- a/packages/opencode/src/altimate/native/finops/databricks-savings.ts +++ b/packages/opencode/src/altimate/native/finops/databricks-savings.ts @@ -9,9 +9,9 @@ import * as core from "@altimateai/altimate-core" import { Flag } from "@opencode-ai/core/flag/flag" import * as Registry from "../connections/registry" -import { isQueryTimeoutError } from "@altimateai/drivers/types" +import { silenceWith, type CoreTelemetrySwitch } from "../core-telemetry" import type { QueryContext } from "../query-execution" -import { RunBudget } from "../../run-budget" +import { reasonOf, savingsRun } from "./savings-run" import { resolveFinopsWarehouse } from "./warehouse-resolver" import type { DatabricksSavingsParams, DatabricksSavingsResult, DatabricksSavingsReport } from "../types" @@ -27,14 +27,18 @@ const WINDOW_END_MS = 6 * 3_600_000 /** Rows read per query; a query that fills it says so in the result. */ const ROW_LIMIT = 10_000 -// The whole check, connecting included, stops after this long; a query still running then is -// cancelled on the warehouse. Claude Code runs a plugin's process for ten minutes at most, so the -// check ends before it is killed. -export const SAVINGS_RUN_LIMIT_MS = 9 * 60_000 + +export { SAVINGS_RUN_LIMIT_MS } from "./savings-run" /** A SQL warehouse's id in a connection's HTTP path (`/sql/1.0/warehouses/`, or the older `/sql/1.0/endpoints/`). */ const WAREHOUSE_PATH = /\/(?:warehouses|endpoints)\/([0-9a-z]+)\/?$/i +/** List price of a usage row: the price in force when the usage started. */ +const LIST_PRICE_JOIN = `LEFT JOIN system.billing.list_prices lp + ON u.sku_name = lp.sku_name AND u.cloud = lp.cloud + AND u.usage_start_time >= lp.price_start_time + AND (lp.price_end_time IS NULL OR u.usage_start_time < lp.price_end_time)` + /** * Per-warehouse usage over the seven days to 06:00 UTC on the parameter, for every warehouse of * every workspace the connection's system tables cover (as Altimate's pipeline reads them), each @@ -112,10 +116,7 @@ bill_rows AS ( lp.pricing.default AS price, u.usage_quantity * lp.pricing.default AS cost FROM system.billing.usage u CROSS JOIN w - LEFT JOIN system.billing.list_prices lp - ON u.sku_name = lp.sku_name AND u.cloud = lp.cloud - AND u.usage_start_time >= lp.price_start_time - AND (lp.price_end_time IS NULL OR u.usage_start_time < lp.price_end_time) + ${LIST_PRICE_JOIN} WHERE u.billing_origin_product = 'SQL' AND u.usage_metadata.warehouse_id IS NOT NULL AND u.workspace_id IN (SELECT workspace_id FROM inv) AND u.usage_metadata.warehouse_id IN (SELECT warehouse_id FROM inv) @@ -156,12 +157,6 @@ WHERE inv.delete_time IS NULL ORDER BY inv.warehouse_name ` -/** List price of a usage row: the price in force when the usage started. */ -const LIST_PRICE_JOIN = `LEFT JOIN system.billing.list_prices lp - ON u.sku_name = lp.sku_name AND u.cloud = lp.cloud - AND u.usage_start_time >= lp.price_start_time - AND (lp.price_end_time IS NULL OR u.usage_start_time < lp.price_end_time)` - /** * Per all-purpose cluster (created from the UI or API, not deleted) of every workspace the * connection's system tables cover: worker node minutes, active days, idle worker minutes (CPU @@ -328,7 +323,7 @@ ORDER BY t.task_hours DESC, t.workspace_id, t.job_id, t.task_key, t.cluster_id, /** Core's rules: a `DatabricksSavingsInput` as JSON in, the report out. */ export type RulesEngine = (input: string) => DatabricksSavingsReport -type CoreModule = { databricksSavings?: unknown; initSdk?: (...args: [string, string, string, boolean, boolean]) => void } +type CoreModule = { databricksSavings?: unknown } & CoreTelemetrySwitch /** * The installed core's rules, or null when its binary predates them. In assessment mode core's own @@ -338,7 +333,7 @@ type CoreModule = { databricksSavings?: unknown; initSdk?: (...args: [string, st export function rulesEngine(lib: CoreModule = core as unknown as CoreModule): RulesEngine | null { const found = lib.databricksSavings if (typeof found !== "function") return null - if (Flag.ALTIMATE_ASSESSMENT_MODE) lib.initSdk?.("", "", "", false, false) + if (Flag.ALTIMATE_ASSESSMENT_MODE) silenceWith(lib) return found as RulesEngine } @@ -429,7 +424,8 @@ function records(result: { columns: string[]; rows: unknown[][] }): Record { - if (one.type.toLowerCase() !== "databricks") return false + // `connections.json` is edited by hand: a `type` that is not a string is not Databricks. + if (typeof one.type !== "string" || one.type.toLowerCase() !== "databricks") return false const config = Registry.getConfig(one.name) as Record | undefined return WAREHOUSE_PATH.test(String(config?.http_path ?? config?.httpPath ?? "")) })?.name @@ -465,57 +461,18 @@ export async function databricksSavings( return failed("This altimate-code build's altimate-core has no Databricks savings rules. Update altimate-code, then try again.") } - // Stopping the session cancels the running query and starts no other. The check stops at its own - // limit, or earlier at the session's query cutoff (`altimate run --time-budget`). - const signal = context?.signal - const ownLimitAt = Date.now() + SAVINGS_RUN_LIMIT_MS - const sessionCutoff = context?.sessionID ? RunBudget.queryDeadline(context.sessionID) : undefined - const stopsAt = Math.min(ownLimitAt, sessionCutoff ?? Infinity) - const limitName = stopsAt < ownLimitAt ? "the run's time budget" : "the check's 9-minute limit" - - // Connecting can wait on a stopped warehouse: it counts toward the limit and yields to a stop. - const beforeStop = async (work: Promise, what: string): Promise => { - signal?.throwIfAborted() - let timer: ReturnType | undefined - let onAbort: (() => void) | undefined - try { - return await Promise.race([ - work, - new Promise((_, reject) => { - timer = setTimeout(() => reject(new Error(`${what} took longer than ${limitName}.`)), Math.max(0, stopsAt - Date.now())) - }), - new Promise((_, reject) => { - onAbort = () => reject(signal!.reason) - signal?.addEventListener("abort", onAbort, { once: true }) - }), - ]) - } finally { - clearTimeout(timer) - if (onAbort) signal?.removeEventListener("abort", onAbort) - work.catch(() => {}) - } - } + const run = savingsRun(context) + const signal = run.signal try { - const connector = await beforeStop(Registry.get(resolved.warehouse), "Connecting to the warehouse") + const connector = await run.beforeStop(() => Registry.get(resolved.warehouse), "Connecting to the warehouse") const binds = [window.end] // A query that fills the row limit reads only its first rows; the result names it. const limited: string[] = [] const read = async (what: string, sql: string, rows = what) => { - signal?.throwIfAborted() - const left = stopsAt - Date.now() - if (left <= 0) throw new Error(`The ${what} query was not started: it would run past ${limitName}.`) - try { - const result = await connector.execute(sql, ROW_LIMIT, binds, { cancelAfterMs: left, signal }) - if (result.truncated) limited.push(rows) - return records(result) - } catch (e) { - if (!isQueryTimeoutError(e)) throw e - const outcome = (e as { cancelled?: boolean }).cancelled - ? "and was cancelled on the warehouse" - : "and may still be running on the warehouse" - throw new Error(`The ${what} query ran past ${limitName} ${outcome}.`) - } + const result = await run.query(connector, what, sql, ROW_LIMIT, binds) + if (result.truncated) limited.push(rows) + return records(result) } const warehouses = await read("SQL warehouses", DATABRICKS_WAREHOUSE_FACTS_SQL) if (warehouses.length === 0) { @@ -528,7 +485,7 @@ export async function databricksSavings( return await read(what, sql, rows) } catch (e) { if (signal?.aborted) throw e - skipped.push({ check, reason: (e instanceof Error ? e.message : String(e)).split("\n")[0]!.slice(0, 300) }) + skipped.push({ check, reason: reasonOf(e) }) return [] } } @@ -548,7 +505,7 @@ export async function databricksSavings( // cluster and job checks are named as skipped instead of losing every finding. if (input.clusters.length === 0 && input.jobs.length === 0) throw e report = rules(JSON.stringify({ ...input, clusters: [], jobs: [] })) - const reason = `core refused their facts: ${(e instanceof Error ? e.message : String(e)).split("\n")[0]!.slice(0, 240)}` + const reason = `core refused their facts: ${reasonOf(e, 240)}` if (input.clusters.length > 0) skipped.push({ check: "Idle Cluster Detection", reason }) if (input.jobs.length > 0) skipped.push({ check: "Job Running on All-Purpose Compute", reason }) clusters.length = 0 diff --git a/packages/opencode/src/altimate/native/finops/savings-run.ts b/packages/opencode/src/altimate/native/finops/savings-run.ts new file mode 100644 index 0000000000..30fc6798e8 --- /dev/null +++ b/packages/opencode/src/altimate/native/finops/savings-run.ts @@ -0,0 +1,70 @@ +// altimate_change - new file +/** + * What the Databricks and Snowflake savings checks share: one time limit for the whole check, connecting + * included, the session's stop, and how a query that runs past either is reported. + */ + +import type { Connector } from "@altimateai/drivers/types" +import { isQueryTimeoutError } from "@altimateai/drivers/types" +import { untilStopped } from "@altimateai/drivers/stop" +import type { QueryContext } from "../query-execution" +import { RunBudget } from "../../run-budget" + +// Claude Code runs a plugin's process for ten minutes at most: the check stops before that, so a query +// still running is cancelled on the warehouse here and not left behind when the process is killed. +export const SAVINGS_RUN_LIMIT_MS = 9 * 60_000 + +/** The limit as the tools and their messages state it. */ +export const SAVINGS_RUN_LIMIT_MINUTES = SAVINGS_RUN_LIMIT_MS / 60_000 + +/** + * One check's run: it stops at its own limit, or earlier at the session's query cutoff (`altimate run + * --time-budget`). Stopping the session cancels the running query and starts no other. + */ +export function savingsRun(context?: QueryContext) { + const signal = context?.signal + const ownLimitAt = Date.now() + SAVINGS_RUN_LIMIT_MS + const sessionCutoff = context?.sessionID ? RunBudget.queryDeadline(context.sessionID) : undefined + const stopsAt = Math.min(ownLimitAt, sessionCutoff ?? Infinity) + const limitName = stopsAt < ownLimitAt ? "the run's time budget" : `the check's ${SAVINGS_RUN_LIMIT_MINUTES}-minute limit` + + /** + * Starts `start` only while the session runs and time is left, then waits on it until either ends. + * Connecting can wait on the network, a stopped warehouse or a sign-in, so it counts toward the limit. + */ + async function beforeStop(start: () => Promise, what: string): Promise { + signal?.throwIfAborted() + const left = stopsAt - Date.now() + if (left <= 0) throw new Error(`${what} was not started: ${limitName} had already passed.`) + const outcome = await untilStopped(start(), { deadlineMs: left, signal }) + if (outcome.stopped === "aborted") throw signal!.reason + if (outcome.stopped) throw new Error(`${what} took longer than ${limitName}.`) + return outcome.value + } + + /** One query with what is left of the limit; the driver cancels it on the warehouse past that, or on a stop. */ + async function query(connector: Connector, what: string, sql: string, rowLimit: number, binds: unknown[]) { + signal?.throwIfAborted() + const left = stopsAt - Date.now() + if (left <= 0) throw new Error(`The ${what} query was not started: it would run past ${limitName}.`) + try { + return await connector.execute(sql, rowLimit, binds, { cancelAfterMs: left, signal }) + } catch (e) { + if (!isQueryTimeoutError(e)) throw e + const { cancelled, unsent } = e as { cancelled?: boolean; unsent?: boolean } + const outcome = unsent + ? "before it was sent, so it did not run on the warehouse" + : cancelled + ? "and was cancelled on the warehouse" + : "and may still be running on the warehouse" + throw new Error(`The ${what} query ran past ${limitName} ${outcome}.`) + } + } + + return { signal, limitName, beforeStop, query } +} + +/** The first line of an error, short enough for a `skipped` entry. */ +export function reasonOf(e: unknown, length = 300): string { + return (e instanceof Error ? e.message : String(e)).split("\n")[0]!.slice(0, length) +} diff --git a/packages/opencode/src/altimate/native/finops/snowflake-savings.ts b/packages/opencode/src/altimate/native/finops/snowflake-savings.ts index ed6fc2d4bc..157de716a0 100644 --- a/packages/opencode/src/altimate/native/finops/snowflake-savings.ts +++ b/packages/opencode/src/altimate/native/finops/snowflake-savings.ts @@ -11,12 +11,12 @@ */ import * as core from "@altimateai/altimate-core" -import { isQueryTimeoutError } from "@altimateai/drivers/types" import { Flag } from "@opencode-ai/core/flag/flag" import * as Registry from "../connections/registry" +import { silenceWith, type CoreTelemetrySwitch } from "../core-telemetry" import type { QueryContext } from "../query-execution" -import { RunBudget } from "../../run-budget" import { savingsWindow } from "./databricks-savings" +import { reasonOf, savingsRun } from "./savings-run" import { resolveFinopsWarehouse } from "./warehouse-resolver" import type { SnowflakeSavingsParams, SnowflakeSavingsReport, SnowflakeSavingsResult } from "../types" @@ -34,9 +34,7 @@ export const SHOW_ROW_LIMIT = 9_999 /** Dollars per credit when the account's rate sheet cannot be read: Enterprise list price. */ export const LIST_PRICE_PER_CREDIT = 3 -// The whole check stops after this long, inside the 10 minutes a Claude Code plugin's process may -// run, so a running query is cancelled here and not left behind when the process is killed. -export const SAVINGS_RUN_LIMIT_MS = 9 * 60_000 +export { SAVINGS_RUN_LIMIT_MS } from "./savings-run" /** The end of the window as a timestamp for filtering `ACCOUNT_USAGE`, and as UTC wall-clock time. */ const WINDOW_END = `TO_TIMESTAMP_TZ(? || ' 00:00:00 +0000', 'YYYY-MM-DD HH24:MI:SS TZHTZM')` @@ -238,13 +236,13 @@ ORDER BY DATE DESC, EFFECTIVE_RATE LIMIT 1 /** Core's rules: a `SnowflakeSavingsInput` as JSON in, the report out. */ export type RulesEngine = (input: string) => SnowflakeSavingsReport -type CoreModule = { snowflakeSavings?: unknown; initSdk?: (...args: [string, string, string, boolean, boolean]) => void } +type CoreModule = { snowflakeSavings?: unknown } & CoreTelemetrySwitch /** The installed core's rules, or null when its binary predates them. See `databricks-savings.ts` for assessment mode. */ export function rulesEngine(lib: CoreModule = core as unknown as CoreModule): RulesEngine | null { const found = lib.snowflakeSavings if (typeof found !== "function") return null - if (Flag.ALTIMATE_ASSESSMENT_MODE) lib.initSdk?.("", "", "", false, false) + if (Flag.ALTIMATE_ASSESSMENT_MODE) silenceWith(lib) return found as RulesEngine } @@ -296,6 +294,8 @@ function timeOf(value: unknown): number { * name it was seen under. A dropped warehouse's name can come back on a new one, so a name keeps its * newest id, and usage that ended before the listed warehouse was created is its predecessor's. * `unlisted` names the warehouses that billed credits but are not listed: dropped, or hidden from the role. + * `replaced` names the listed ones whose name an earlier warehouse also billed credits under: that + * earlier usage is left out. */ export function buildFacts(rows: FactRows) { const listed = new Set(rows.warehouses.map((row) => String(row.name ?? ""))) @@ -325,6 +325,11 @@ export function buildFacts(rows: FactRows) { return byName } const metering = of(rows.metering) + const replaced = new Set() + for (const row of rows.metering) { + const name = nameOf(row) + if (listed.has(name) && newestId.get(name) !== num(row.warehouse_id) && num(row.credits_used) > 0) replaced.add(name) + } const periods = of(rows.periods) const restarts = of(rows.restarts) const queries = of(rows.queries) @@ -342,6 +347,7 @@ export function buildFacts(rows: FactRows) { const facts = rows.warehouses.map((row) => { const name = String(row.name ?? "") const id = idFor(row) + if (id === undefined && (metering.get(name) ?? []).some((one) => num(one.credits_used) > 0)) replaced.add(name) const usedBy = (byName: Map) => (id === undefined ? [] : (byName.get(name) ?? [])) const usage = new Map() const idle = new Map() @@ -411,7 +417,7 @@ export function buildFacts(rows: FactRows) { } }) const unlisted = [...metering.keys()].filter((name) => !listed.has(name)).sort() - return { facts, unlisted } + return { facts, unlisted, replaced: [...replaced].sort() } } /** The price of a credit: the account's rate when it can be read in dollars, otherwise list price. */ @@ -443,58 +449,18 @@ export async function snowflakeSavings( return failed("This altimate-code build's altimate-core has no Snowflake savings rules. Update altimate-code, then try again.") } - // Stopping the session cancels the running query and starts no other. The check stops at its own - // limit, or earlier at the session's query cutoff (`altimate run --time-budget`). - const signal = context?.signal - const ownLimitAt = Date.now() + SAVINGS_RUN_LIMIT_MS - const sessionCutoff = context?.sessionID ? RunBudget.queryDeadline(context.sessionID) : undefined - const stopsAt = Math.min(ownLimitAt, sessionCutoff ?? Infinity) - const limitName = stopsAt < ownLimitAt ? "the run's time budget" : "the check's 9-minute limit" - - // Connecting can wait on the network or a sign-in: it counts toward the limit and yields to a stop. - const beforeStop = async (work: Promise, what: string): Promise => { - // A stop can come first, and a later failure of `work` must not go unhandled. - work.catch(() => {}) - signal?.throwIfAborted() - let timer: ReturnType | undefined - let onAbort: (() => void) | undefined - try { - return await Promise.race([ - work, - new Promise((_, reject) => { - timer = setTimeout(() => reject(new Error(`${what} took longer than ${limitName}.`)), Math.max(0, stopsAt - Date.now())) - }), - new Promise((_, reject) => { - onAbort = () => reject(signal!.reason) - signal?.addEventListener("abort", onAbort, { once: true }) - }), - ]) - } finally { - clearTimeout(timer) - if (onAbort) signal?.removeEventListener("abort", onAbort) - } - } + const run = savingsRun(context) + const signal = run.signal try { - const connector = await beforeStop(Registry.get(resolved.warehouse), "Connecting to the warehouse") + const connector = await run.beforeStop(() => Registry.get(resolved.warehouse), "Connecting to the warehouse") const config = Registry.getConfig(resolved.warehouse) as Record | undefined const limited: string[] = [] const read = async (what: string, sql: string, rowLimit = ROW_LIMIT) => { - signal?.throwIfAborted() - const left = stopsAt - Date.now() - if (left <= 0) throw new Error(`The ${what} query was not started: it would run past ${limitName}.`) const binds = Array.from({ length: sql.split("?").length - 1 }, () => window.end) - try { - const result = await connector.execute(sql, rowLimit, binds, { cancelAfterMs: left, signal }) - if (result.truncated) limited.push(what) - return records(result) - } catch (e) { - if (!isQueryTimeoutError(e)) throw e - const outcome = (e as { cancelled?: boolean }).cancelled - ? "and was cancelled on the warehouse" - : "and may still be running on the warehouse" - throw new Error(`The ${what} query ran past ${limitName} ${outcome}.`) - } + const result = await run.query(connector, what, sql, rowLimit, binds) + if (result.truncated) limited.push(what) + return records(result) } const warehouses = await read("warehouses", SNOWFLAKE_WAREHOUSES_SQL, SHOW_ROW_LIMIT) if (warehouses.length === 0) return failed("SHOW WAREHOUSES lists no warehouse this connection's role can see.") @@ -502,7 +468,6 @@ export async function snowflakeSavings( // A view the role may not read stops only the checks that need it; the rest still report. const skipped: { check: string; reason: string }[] = [] let checksMissing = 0 - const reasonOf = (e: unknown) => (e instanceof Error ? e.message : String(e)).split("\n")[0]!.slice(0, 300) const optional = async (check: string, what: string, sql: string) => { try { return await read(what, sql) @@ -513,13 +478,6 @@ export async function snowflakeSavings( return [] } } - // Most accounts cannot read the rate sheet (it needs organization access); list price stands in. - const rate = await read("credit rate", SNOWFLAKE_RATE_SQL).catch((e) => { - if (signal?.aborted) throw e - // An unreadable rate sheet is ordinary; a query that may still be running is worth naming. - if (reasonOf(e).includes("may still be running")) skipped.push({ check: "The account's credit rate", reason: reasonOf(e) }) - return [] as Row[] - }) const metering = await read("metering", SNOWFLAKE_METERING_FACTS_SQL) const periods = await optional( "High Idle Time, Underutilized Cluster and the short-period signal of Downsize Warehouse", @@ -536,14 +494,28 @@ export async function snowflakeSavings( return failed(`No check could run. ${reason.reason}`) } + // Read last: only the price needs it, and a slow or unreadable rate sheet must not cost a usage + // query its time. Most accounts cannot read it (it needs organization access); list price stands in. + const rate = await read("credit rate", SNOWFLAKE_RATE_SQL).catch((e) => { + if (signal?.aborted) throw e + // An unreadable rate sheet is ordinary; a query that may still be running is worth naming. + if (reasonOf(e).includes("may still be running")) skipped.push({ check: "The account's credit rate", reason: reasonOf(e) }) + return [] as Row[] + }) const price = creditPrice(rate[0]) - const { facts, unlisted } = buildFacts({ warehouses, metering, periods, restarts, queries }) + const { facts, unlisted, replaced } = buildFacts({ warehouses, metering, periods, restarts, queries }) if (unlisted.length > 0) { skipped.push({ check: `${unlisted.length} warehouse${unlisted.length === 1 ? "" : "s"} that billed credits`, reason: `SHOW WAREHOUSES does not list ${unlisted.slice(0, 5).join(", ")}${unlisted.length > 5 ? ", …" : ""}: dropped, renamed, or not visible to the connection's role`, }) } + if (replaced.length > 0) { + skipped.push({ + check: `Earlier usage of ${replaced.length} warehouse name${replaced.length === 1 ? "" : "s"}`, + reason: `${replaced.slice(0, 5).join(", ")}${replaced.length > 5 ? ", …" : ""} also billed credits as an earlier warehouse, since dropped or renamed: only the current warehouse's usage was checked`, + }) + } const report = rules(JSON.stringify({ lookback_days: LOOKBACK_DAYS, price_per_credit: price.per_credit, warehouses: facts })) return { success: true, diff --git a/packages/opencode/src/altimate/tools/finops-databricks-savings.ts b/packages/opencode/src/altimate/tools/finops-databricks-savings.ts index 867217f35c..7ec606719f 100644 --- a/packages/opencode/src/altimate/tools/finops-databricks-savings.ts +++ b/packages/opencode/src/altimate/tools/finops-databricks-savings.ts @@ -3,6 +3,8 @@ import z from "zod" import { Tool } from "../../tool/tool" import { Dispatcher } from "../native" import type { DatabricksSavingsFinding, DatabricksSavingsReport, DatabricksSavingsResult } from "../native/types" +import { SAVINGS_RUN_LIMIT_MINUTES } from "../native/finops/savings-run" +import { dollars, oneLine, range } from "./finops-formatting" /** One shape for every outcome, so a reader of the metadata never guesses which fields exist. */ type SavingsMetadata = { @@ -23,16 +25,6 @@ type SavingsMetadata = { needs_confirmation?: boolean } -export function dollars(value: number): string { - return value >= 100 - ? `$${Math.round(value).toLocaleString("en-US")}` - : `$${value.toFixed(2)}` -} - -export function range(low: number, high: number): string { - return dollars(low) === dollars(high) ? dollars(high) : `${dollars(low)}–${dollars(high)}` -} - function percent(share: unknown): string { return `${Math.round(Number(share) * 100)}%` } @@ -47,12 +39,11 @@ const percentOf = (value: unknown) => `${Math.round(Number(value))}%` const tenths = (value: unknown) => Number(value) < 0.05 ? "under 0.1" : Number(value).toLocaleString("en-US", { minimumFractionDigits: 1, maximumFractionDigits: 1 }) -/** Free text from the workspace on one line, with control and invisible characters gone. */ -export const oneLine = (value: unknown) => - String(value ?? "") - .replace(/\p{Cf}/gu, "") - .replace(/[\s\p{Cc}]+/gu, " ") - .trim() +/** An estimate to one decimal: "about 1.2", or "under 0.1". */ +const aboutTenths = (value: unknown) => { + const text = tenths(value) + return text === "under 0.1" ? text : `about ${text}` +} /** What a finding rests on, one short line each. */ export function evidenceLines(finding: DatabricksSavingsFinding): string[] { @@ -76,7 +67,7 @@ export function evidenceLines(finding: DatabricksSavingsFinding): string[] { ] case "job_on_all_purpose": return [ - `${Number(e.task_runs).toLocaleString("en-US")} task ${Number(e.task_runs) === 1 ? "run" : "runs"} (${String(e.trigger_type).toLowerCase()}), ${tenths(e.task_hours)} hours on all-purpose compute, ${Number(e.estimated_dbus) < 0.05 ? "under 0.1" : `about ${tenths(e.estimated_dbus)}`} DBUs, most on cluster ${JSON.stringify(oneLine(e.host_cluster_name) || oneLine(e.host_cluster_id))}`, + `${Number(e.task_runs).toLocaleString("en-US")} task ${Number(e.task_runs) === 1 ? "run" : "runs"} (${String(e.trigger_type).toLowerCase()}), ${tenths(e.task_hours)} hours on all-purpose compute, ${aboutTenths(e.estimated_dbus)} DBUs, most on cluster ${JSON.stringify(oneLine(e.host_cluster_name) || oneLine(e.host_cluster_id))}`, `$${Number(e.all_purpose_rate_per_dbu).toFixed(3)} a DBU now; a job cluster would be about $${Number(e.jobs_rate_per_dbu).toFixed(3)}${e.jobs_rate_estimated ? " (estimated: the workspace billed no jobs compute)" : ""}`, ] default: @@ -131,7 +122,7 @@ export const FinopsDatabricksSavingsTool = Tool.define("finops_databricks_saving .boolean() .optional() .describe( - "Set to true only after the user has agreed to run the check. It runs three queries on their SQL warehouse, which Databricks starts if it is stopped and bills while it runs; the check stops after 9 minutes, cancelling any query still running.", + `Set to true only after the user has agreed to run the check. It runs three queries on their SQL warehouse, which Databricks starts if it is stopped and bills while it runs; the check stops after ${SAVINGS_RUN_LIMIT_MINUTES} minutes, cancelling any query still running.`, ), }), async execute(args, ctx) { @@ -142,7 +133,7 @@ export const FinopsDatabricksSavingsTool = Tool.define("finops_databricks_saving metadata: { success: true, needs_confirmation: true } as SavingsMetadata, output: "Not run. The Databricks savings check runs three queries over seven days of system tables on the connection's SQL warehouse, " + - "which Databricks starts if it is stopped and bills while it runs. It stops after 9 minutes, cancelling any query still running. " + + `which Databricks starts if it is stopped and bills while it runs. It stops after ${SAVINGS_RUN_LIMIT_MINUTES} minutes, cancelling any query still running. ` + "Ask the user whether to run it; if they agree, call this tool again with confirm: true.", } } diff --git a/packages/opencode/src/altimate/tools/finops-formatting.ts b/packages/opencode/src/altimate/tools/finops-formatting.ts index c88aa5579a..63d61abe16 100644 --- a/packages/opencode/src/altimate/tools/finops-formatting.ts +++ b/packages/opencode/src/altimate/tools/finops-formatting.ts @@ -17,3 +17,20 @@ export function truncateQuery(text: string, maxLen: number): string { if (oneLine.length <= maxLen) return oneLine return oneLine.slice(0, maxLen - 3) + "..." } + +/** Dollars to the cent below $100, to the dollar from there. */ +export function dollars(value: number): string { + return value >= 100 ? `$${Math.round(value).toLocaleString("en-US")}` : `$${value.toFixed(2)}` +} + +/** A low–high dollar range, or one figure when both read the same. */ +export function range(low: number, high: number): string { + return dollars(low) === dollars(high) ? dollars(high) : `${dollars(low)}–${dollars(high)}` +} + +/** Free text from the warehouse on one line, with control and invisible characters gone. */ +export const oneLine = (value: unknown) => + String(value ?? "") + .replace(/\p{Cf}/gu, "") + .replace(/[\s\p{Cc}]+/gu, " ") + .trim() diff --git a/packages/opencode/src/altimate/tools/finops-snowflake-savings.ts b/packages/opencode/src/altimate/tools/finops-snowflake-savings.ts index a748cd6c38..45c105b9e1 100644 --- a/packages/opencode/src/altimate/tools/finops-snowflake-savings.ts +++ b/packages/opencode/src/altimate/tools/finops-snowflake-savings.ts @@ -3,7 +3,8 @@ import z from "zod" import { Tool } from "../../tool/tool" import { Dispatcher } from "../native" import type { SnowflakeSavingsFinding, SnowflakeSavingsReport, SnowflakeSavingsResult } from "../native/types" -import { dollars, oneLine, range } from "./finops-databricks-savings" +import { SAVINGS_RUN_LIMIT_MINUTES } from "../native/finops/savings-run" +import { oneLine, range } from "./finops-formatting" /** One shape for every outcome, so a reader of the metadata never guesses which fields exist. */ type SavingsMetadata = { @@ -116,7 +117,7 @@ export const FinopsSnowflakeSavingsTool = Tool.define("finops_snowflake_savings" .boolean() .optional() .describe( - "Set to true only after the user has agreed to run the check. It runs six queries on their Snowflake account; five use the connection's warehouse, which Snowflake resumes if it is suspended and auto-resume is on, and which bills while it runs; the check stops after 9 minutes, cancelling any query still running.", + `Set to true only after the user has agreed to run the check. It runs six queries on their Snowflake account; five use the connection's warehouse, which Snowflake resumes if it is suspended and auto-resume is on, and which bills while it runs; the check stops after ${SAVINGS_RUN_LIMIT_MINUTES} minutes, cancelling any query still running.`, ), }), async execute(args, ctx) { @@ -127,7 +128,7 @@ export const FinopsSnowflakeSavingsTool = Tool.define("finops_snowflake_savings" metadata: { success: true, needs_confirmation: true } as SavingsMetadata, output: "Not run. The Snowflake savings check runs six queries that read up to eleven days of account usage, to report on seven; five use the connection's warehouse, " + - "which Snowflake resumes if it is suspended and auto-resume is on, and which bills while it runs. It stops after 9 minutes, cancelling any query still running. " + + `which Snowflake resumes if it is suspended and auto-resume is on, and which bills while it runs. It stops after ${SAVINGS_RUN_LIMIT_MINUTES} minutes, cancelling any query still running. ` + "Ask the user whether to run it; if they agree, call this tool again with confirm: true.", } } diff --git a/packages/opencode/test/altimate/finops-databricks-e2e.test.ts b/packages/opencode/test/altimate/finops-databricks-e2e.test.ts index 1584efc472..c56805d6a9 100644 --- a/packages/opencode/test/altimate/finops-databricks-e2e.test.ts +++ b/packages/opencode/test/altimate/finops-databricks-e2e.test.ts @@ -215,6 +215,11 @@ describe.skipIf(!HAS_DATABRICKS)("Databricks FinOps Integration", () => { expect(result.error).toBeUndefined() expect(result.success).toBe(true) expect(result.warehouses_checked).toBeGreaterThan(0) + // The cluster and job queries ran against the real system tables: a broken query would + // only show up as a skipped check, never as a failure. + expect(result.skipped).toEqual([]) + expect(typeof result.clusters_checked).toBe("number") + expect(typeof result.jobs_checked).toBe("number") expect(Array.isArray(result.report!.findings)).toBe(true) expect(result.report!.total.resources).toBe( new Set(result.report!.findings.map((f) => `${f.resource_type}:${f.workspace_id ?? ""}:${f.resource_id}`)).size, diff --git a/packages/opencode/test/altimate/finops-databricks-savings.test.ts b/packages/opencode/test/altimate/finops-databricks-savings.test.ts index bb39f365fa..f2a823071c 100644 --- a/packages/opencode/test/altimate/finops-databricks-savings.test.ts +++ b/packages/opencode/test/altimate/finops-databricks-savings.test.ts @@ -18,6 +18,13 @@ import { import { FinopsDatabricksSavingsTool, formatSavings } from "../../src/altimate/tools/finops-databricks-savings" import type { DatabricksSavingsResult } from "../../src/altimate/native/types" +/** Every spy a test installs, restored after it whether or not its assertions passed. */ +const spies: { mockRestore(): void }[] = [] +const tracked = (spy: T): T => (spies.push(spy), spy) +afterEach(() => { + for (const spy of spies.splice(0).reverse()) spy.mockRestore() +}) + function stubCtx(): any { return { sessionID: "test", messageID: "test", agent: "test", abort: new AbortController().signal, metadata: () => {}, ask: async () => {} } } @@ -269,11 +276,10 @@ describe("databricks savings handler", () => { Registry.setConfigs({ dbx: { type: "databricks", server_hostname: "dbc-0.cloud.databricks.com", http_path: "/sql/1.0/warehouses/abc123", access_token: "x" }, }) - const get = spyOn(Registry, "get").mockRejectedValue(new Error("opened a connection")) + const get = tracked(spyOn(Registry, "get")).mockRejectedValue(new Error("opened a connection")) // A published core that predates the rules exports `initSdk` but not `databricksSavings`. const result = await databricksSavings({ warehouse: "dbx" }, rulesEngine({ initSdk: () => {} })) const connections = get.mock.calls.length - get.mockRestore() expect(result.success).toBe(false) expect(result.error).toBe( "This altimate-code build's altimate-core has no Databricks savings rules. Update altimate-code, then try again.", @@ -460,7 +466,7 @@ describe("databricks savings handler over a stubbed connection", () => { } const connect = (answer: (sql: string) => unknown) => - spyOn(Registry, "get").mockResolvedValue({ + tracked(spyOn(Registry, "get")).mockResolvedValue({ execute: async (sql: string, limit: number, binds: unknown[], options?: { cancelAfterMs?: number; queryTimeoutMs?: number; signal?: AbortSignal }) => { calls.push({ sql, limit, binds, timeout: options?.cancelAfterMs, clientTimeout: options?.queryTimeoutMs, signal: options?.signal }) return answer(sql) @@ -473,7 +479,6 @@ describe("databricks savings handler over a stubbed connection", () => { return sql === DATABRICKS_WAREHOUSE_FACTS_SQL ? table(WAREHOUSE_ROW) : table(JOB_ROW) }) const result = await databricksSavings({ warehouse: "dbx", now_ms: Date.parse("2026-11-18T10:00:00Z") }, rules) - spy.mockRestore() expect(result.success).toBe(true) expect(result.skipped).toEqual([{ check: "Idle Cluster Detection", reason: "[INSUFFICIENT_PERMISSIONS] system.compute.node_timeline" }]) expect([result.warehouses_checked, result.clusters_checked, result.jobs_checked]).toEqual([1, 0, 1]) @@ -506,7 +511,6 @@ describe("databricks savings handler over a stubbed connection", () => { : { columns: [], rows: [], truncated: false }, ) const result = await databricksSavings({ warehouse: "dbx", now_ms: Date.parse("2026-11-18T10:00:00Z") }, rules) - spy.mockRestore() expect(sent.jobs).toHaveLength(3) expect(result.jobs_checked).toBe(2) expect(result.success).toBe(true) @@ -522,29 +526,28 @@ describe("databricks savings handler over a stubbed connection", () => { : { columns: [], rows: [], truncated: false }, ) const result = await databricksSavings({ warehouse: "dbx", now_ms: Date.parse("2026-11-18T10:00:00Z") }, rules) - spy.mockRestore() expect(result.success).toBe(true) expect(result.jobs_checked).toBe(2) }) - test("a query past its limit fails the check with a plain reason, or skips an optional check", async () => { - const timeout = (cancelled: boolean) => Object.assign(new Error("QUERY_TIMEOUT: databricks query exceeded"), { code: "QUERY_TIMEOUT", cancelled }) - let spy = connect((sql) => { + const timeout = (cancelled: boolean) => Object.assign(new Error("QUERY_TIMEOUT: databricks query exceeded"), { code: "QUERY_TIMEOUT", cancelled }) + + test("a required query past its limit fails the check with a plain reason", async () => { + connect((sql) => { if (sql === DATABRICKS_WAREHOUSE_FACTS_SQL) throw timeout(true) return table(JOB_ROW) }) - let result = await databricksSavings({ warehouse: "dbx", now_ms: Date.parse("2026-11-18T10:00:00Z") }, rules) - spy.mockRestore() + const result = await databricksSavings({ warehouse: "dbx", now_ms: Date.parse("2026-11-18T10:00:00Z") }, rules) expect(result.success).toBe(false) expect(result.error).toBe("The SQL warehouses query ran past the check's 9-minute limit and was cancelled on the warehouse.") + }) - calls = [] - spy = connect((sql) => { + test("an optional query past its limit skips only its check", async () => { + connect((sql) => { if (sql === DATABRICKS_JOB_FACTS_SQL) throw timeout(false) return sql === DATABRICKS_WAREHOUSE_FACTS_SQL ? table(WAREHOUSE_ROW) : table({ cluster_id: "c1" }) }) - result = await databricksSavings({ warehouse: "dbx", now_ms: Date.parse("2026-11-18T10:00:00Z") }, rules) - spy.mockRestore() + const result = await databricksSavings({ warehouse: "dbx", now_ms: Date.parse("2026-11-18T10:00:00Z") }, rules) expect(result.success).toBe(true) expect(result.skipped).toEqual([ { check: "Job Running on All-Purpose Compute", reason: "The jobs query ran past the check's 9-minute limit and may still be running on the warehouse." }, @@ -554,15 +557,13 @@ describe("databricks savings handler over a stubbed connection", () => { test("a query is not started once the check's 9 minutes are used up", async () => { const realNow = Date.now let now = realNow() - const clock = spyOn(Date, "now").mockImplementation(() => now) + const clock = tracked(spyOn(Date, "now")).mockImplementation(() => now) const spy = connect((sql) => { // The warehouse query takes the whole limit. if (sql === DATABRICKS_WAREHOUSE_FACTS_SQL) now += SAVINGS_RUN_LIMIT_MS return table(WAREHOUSE_ROW) }) const result = await databricksSavings({ warehouse: "dbx", now_ms: Date.parse("2026-11-18T10:00:00Z") }, rules) - spy.mockRestore() - clock.mockRestore() expect(calls.map((one) => one.sql)).toEqual([DATABRICKS_WAREHOUSE_FACTS_SQL]) expect(result.success).toBe(true) expect(result.skipped).toEqual([ @@ -582,7 +583,6 @@ describe("databricks savings handler over a stubbed connection", () => { return rows(JOB_ROW, { ...JOB_ROW, task_key: "other" }, { ...JOB_ROW, job_id: "j2", workspace_id: null }) }) const result = await databricksSavings({ warehouse: "dbx", now_ms: Date.parse("2026-11-18T10:00:00Z") }, rules) - spy.mockRestore() expect(result.success).toBe(true) expect(result.report?.findings).toEqual([]) expect([result.warehouses_checked, result.clusters_checked, result.jobs_checked]).toEqual([1, 2, 2]) @@ -591,21 +591,34 @@ describe("databricks savings handler over a stubbed connection", () => { test("connecting counts toward the limit: a connection that never comes up ends the check in time", async () => { // A 30 ms session budget stands in for the last moments of the 9 minutes. - const budget = spyOn(RunBudget, "queryDeadline").mockReturnValue(Date.now() + 30) - const get = spyOn(Registry, "get").mockReturnValue(new Promise(() => {}) as any) + const budget = tracked(spyOn(RunBudget, "queryDeadline")).mockReturnValue(Date.now() + 30) + const get = tracked(spyOn(Registry, "get")).mockReturnValue(new Promise(() => {}) as any) const result = await databricksSavings({ warehouse: "dbx", now_ms: Date.parse("2026-11-18T10:00:00Z") }, rules, { sessionID: "s1" }) - get.mockRestore() - budget.mockRestore() expect(result.success).toBe(false) expect(result.error).toBe("Connecting to the warehouse took longer than the run's time budget.") }) + test("a session that already stopped does not connect to the warehouse at all", async () => { + const controller = new AbortController() + controller.abort(new Error("session stopped")) + const get = tracked(spyOn(Registry, "get")).mockRejectedValue(new Error("connected")) + const result = await databricksSavings({ warehouse: "dbx", now_ms: Date.parse("2026-11-18T10:00:00Z") }, rules, { signal: controller.signal }) + expect(result).toMatchObject({ success: false, error: "session stopped" }) + expect(get).not.toHaveBeenCalled() + }) + + test("a session time budget already used up does not connect to the warehouse at all", async () => { + tracked(spyOn(RunBudget, "queryDeadline")).mockReturnValue(Date.now() - 1) + const get = tracked(spyOn(Registry, "get")).mockRejectedValue(new Error("connected")) + const result = await databricksSavings({ warehouse: "dbx", now_ms: Date.parse("2026-11-18T10:00:00Z") }, rules, { sessionID: "s1" }) + expect(result).toMatchObject({ success: false, error: "Connecting to the warehouse was not started: the run's time budget had already passed." }) + expect(get).not.toHaveBeenCalled() + }) + test("a session time budget shorter than 9 minutes is the limit", async () => { - const budget = spyOn(RunBudget, "queryDeadline").mockReturnValue(Date.now() + 60_000) + const budget = tracked(spyOn(RunBudget, "queryDeadline")).mockReturnValue(Date.now() + 60_000) const spy = connect((sql) => (sql === DATABRICKS_WAREHOUSE_FACTS_SQL ? table(WAREHOUSE_ROW) : table(JOB_ROW))) await databricksSavings({ warehouse: "dbx", now_ms: Date.parse("2026-11-18T10:00:00Z") }, rules, { sessionID: "s1" }) - spy.mockRestore() - budget.mockRestore() for (const one of calls) expect(one.timeout).toBeLessThanOrEqual(60_000) }) @@ -619,7 +632,6 @@ describe("databricks savings handler over a stubbed connection", () => { return table(WAREHOUSE_ROW) }) const result = await databricksSavings({ warehouse: "dbx", now_ms: Date.parse("2026-11-18T10:00:00Z") }, rules, { signal: controller.signal }) - spy.mockRestore() expect(result.success).toBe(false) expect(result.error).toBe("session stopped") // Each query got the session's signal, so the driver cancels it on the warehouse. @@ -631,7 +643,6 @@ describe("databricks savings handler over a stubbed connection", () => { test("a query that fills the row limit is named, and the tool says so", async () => { const spy = connect((sql) => (sql === DATABRICKS_WAREHOUSE_FACTS_SQL ? table(WAREHOUSE_ROW) : sql === DATABRICKS_JOB_FACTS_SQL ? table(JOB_ROW, true) : table({ cluster_id: "c1" }))) const result = await databricksSavings({ warehouse: "dbx", now_ms: Date.parse("2026-11-18T10:00:00Z") }, rules) - spy.mockRestore() expect(calls.every((one) => one.limit === 10_000)).toBe(true) expect(result.limited).toEqual(["job task rows"]) expect(formatSavings(result)).toContain("Only the first 10,000 job task rows were read; any beyond them were not checked.") @@ -647,7 +658,6 @@ describe("databricks savings handler over a stubbed connection", () => { return rules(input) } const result = await databricksSavings({ warehouse: "dbx", now_ms: Date.parse("2026-11-18T10:00:00Z") }, picky) - spy.mockRestore() expect(result.success).toBe(true) expect(calls.length).toBe(2) expect(sent.warehouses[0].warehouse_id).toBe("abc123") @@ -663,7 +673,17 @@ describe("databricks savings handler over a stubbed connection", () => { }) const spy = connect((sql) => (sql === DATABRICKS_WAREHOUSE_FACTS_SQL ? table(WAREHOUSE_ROW) : { columns: [], rows: [], truncated: false })) const result = await databricksSavings({ warehouse: " ", now_ms: Date.parse("2026-11-18T10:00:00Z") }, rules) - spy.mockRestore() + expect(result.success).toBe(true) + expect(result.warehouse).toBe("dbx") + }) + + test("a hand-edited connection whose type is not text is passed over, not a crash", async () => { + Registry.setConfigs({ + odd: { type: 42, server_hostname: "x", http_path: "/sql/1.0/warehouses/zzz", access_token: "x" }, + dbx: { type: "databricks", server_hostname: "dbc-0.cloud.databricks.com", http_path: "/sql/1.0/warehouses/abc123", access_token: "x" }, + } as any) + connect((sql) => (sql === DATABRICKS_WAREHOUSE_FACTS_SQL ? table(WAREHOUSE_ROW) : { columns: [], rows: [], truncated: false })) + const result = await databricksSavings({ now_ms: Date.parse("2026-11-18T10:00:00Z") }, rules) expect(result.success).toBe(true) expect(result.warehouse).toBe("dbx") }) @@ -680,7 +700,7 @@ describe("every workspace", () => { Registry.setConfigs({ dbx: { type: "databricks", server_hostname: "dbc-0.cloud.databricks.com", http_path: "/sql/1.0/endpoints/abc123", access_token: "x" }, }) - const spy = spyOn(Registry, "get").mockResolvedValue({ + const spy = tracked(spyOn(Registry, "get")).mockResolvedValue({ execute: async () => ({ columns: ["warehouse_id"], rows: [["abc123"]], truncated: false }), } as any) const result = await databricksSavings({ warehouse: "dbx", now_ms: Date.parse("2026-11-18T10:00:00Z") }, () => ({ @@ -689,7 +709,6 @@ describe("every workspace", () => { total: { monthly_min: 0, monthly_max: 0, resources: 0 }, what_if: [], })) - spy.mockRestore() Registry.reset() expect(result.success).toBe(true) expect(result.ran_on).toBe("abc123") diff --git a/packages/opencode/test/altimate/finops-snowflake-savings.test.ts b/packages/opencode/test/altimate/finops-snowflake-savings.test.ts index 22674f5954..7ff46835f1 100644 --- a/packages/opencode/test/altimate/finops-snowflake-savings.test.ts +++ b/packages/opencode/test/altimate/finops-snowflake-savings.test.ts @@ -22,6 +22,13 @@ import { FinopsSnowflakeSavingsTool, formatSavings } from "../../src/altimate/to import type { SnowflakeSavingsResult } from "../../src/altimate/native/types" import { RunBudget } from "../../src/altimate/run-budget" +/** Every spy a test installs, restored after it whether or not its assertions passed. */ +const spies: { mockRestore(): void }[] = [] +const tracked = (spy: T): T => (spies.push(spy), spy) +afterEach(() => { + for (const spy of spies.splice(0).reverse()) spy.mockRestore() +}) + function stubCtx(): any { return { sessionID: "test", messageID: "test", agent: "test", abort: new AbortController().signal, metadata: () => {}, ask: async () => {} } } @@ -324,6 +331,23 @@ describe("building the facts", () => { expect(facts[0]!.usage_days).toEqual([{ day: "2026-11-12", credits_used: 1, active_periods: 0, short_periods: 0 }]) }) + test("the earlier warehouse's billed usage under a reused name is named, not dropped silently", () => { + const billed = (credits: number) => + buildFacts({ + warehouses: [show()], + metering: [ + { warehouse_id: 7, warehouse_name: "BI_WH", usage_day: "2026-11-11", idle_day: null, credits_used: credits }, + { warehouse_id: 9, warehouse_name: "BI_WH", usage_day: "2026-11-12", idle_day: null, credits_used: 1 }, + ], + periods: [], + restarts: [], + queries: [], + }) + expect(billed(100)).toMatchObject({ unlisted: [], replaced: ["BI_WH"] }) + // An earlier warehouse that billed nothing is nothing left out. + expect(billed(0).replaced).toEqual([]) + }) + const meter = (id: number, name: string, day: string, credits = 1) => ({ warehouse_id: id, warehouse_name: name, usage_day: day, idle_day: null, credits_used: credits }) const none = { periods: [], restarts: [], queries: [] } @@ -406,6 +430,8 @@ describe("building the facts", () => { const recreated = buildFacts({ warehouses: [show({ created_on: new Date("2026-11-15T08:00:00Z") })], ...rows }) expect(recreated.facts[0]).toMatchObject({ warehouse_id: "unused:BI_WH", usage_days: [] }) expect(recreated.unlisted).toEqual([]) + // That usage is named as not checked. + expect(recreated.replaced).toEqual(["BI_WH"]) const older = buildFacts({ warehouses: [show({ created_on: "2026-01-01T00:00:00Z" })], ...rows }) expect(older.facts[0]!.warehouse_id).toBe("100") // A creation time that cannot be read changes nothing. @@ -461,7 +487,7 @@ describe("snowflake savings handler over a stubbed connection", () => { return { lookback_days: 7, price_per_credit: sent.price_per_credit, findings: [], total: { monthly_min: 0, monthly_max: 0, resources: 0 }, what_if: [] } } const connect = (answer: (sql: string) => unknown) => - spyOn(Registry, "get").mockResolvedValue({ + tracked(spyOn(Registry, "get")).mockResolvedValue({ execute: async (sql: string, limit: number, binds: unknown[], options?: { cancelAfterMs?: number; queryTimeoutMs?: number; signal?: AbortSignal }) => { calls.push({ sql, limit, binds, timeout: options?.cancelAfterMs, clientTimeout: options?.queryTimeoutMs, signal: options?.signal }) return answer(sql) @@ -471,16 +497,16 @@ describe("snowflake savings handler over a stubbed connection", () => { test("every query is bound to the window end and gets what is left of the 9 minutes; the facts and rate reach core", async () => { const spy = connect(answers) const result = await snowflakeSavings({ warehouse: "sf", now_ms: NOW }, rules) - spy.mockRestore() expect(result.success).toBe(true) expect(result.window).toEqual({ start: "2026-11-11", end: "2026-11-18" }) + // The rate sheet is read last: only the price needs it. expect(calls.map((one) => one.sql)).toEqual([ SNOWFLAKE_WAREHOUSES_SQL, - SNOWFLAKE_RATE_SQL, SNOWFLAKE_METERING_FACTS_SQL, SNOWFLAKE_PERIOD_FACTS_SQL, SNOWFLAKE_RESTART_FACTS_SQL, SNOWFLAKE_QUERY_FACTS_SQL, + SNOWFLAKE_RATE_SQL, ]) for (const one of calls) { expect(one.binds).toEqual(Array(one.sql.split("?").length - 1).fill("2026-11-18")) @@ -505,7 +531,6 @@ describe("snowflake savings handler over a stubbed connection", () => { return answers(sql) }) const result = await snowflakeSavings({ warehouse: "sf", now_ms: NOW }, rules) - spy.mockRestore() expect(result.success).toBe(true) expect(sent.price_per_credit).toBe(LIST_PRICE_PER_CREDIT) expect(result.price).toEqual({ per_credit: LIST_PRICE_PER_CREDIT, source: "list_price" }) @@ -518,20 +543,18 @@ describe("snowflake savings handler over a stubbed connection", () => { return answers(sql) }) const result = await snowflakeSavings({ warehouse: "sf", now_ms: NOW }, rules) - spy.mockRestore() expect(result.success).toBe(true) expect(result.skipped).toEqual([{ check: "Frequent Restart Overcharge", reason: "not authorized" }]) }) test("when no usage query could run, one that may still be running is the one named", async () => { - const timeout = Object.assign(new Error("QUERY_TIMEOUT"), { code: "QUERY_TIMEOUT", cancelled: false }) + const stopped = timeout({ cancelled: false }) const spy = connect((sql) => { - if (sql === SNOWFLAKE_PERIOD_FACTS_SQL) throw timeout + if (sql === SNOWFLAKE_PERIOD_FACTS_SQL) throw stopped if ([SNOWFLAKE_RESTART_FACTS_SQL, SNOWFLAKE_QUERY_FACTS_SQL].includes(sql)) throw new Error("not authorized") return answers(sql) }) const result = await snowflakeSavings({ warehouse: "sf", now_ms: NOW }, rules) - spy.mockRestore() expect(result.success).toBe(false) expect(result.error).toContain("The active periods query ran past") expect(result.error).toContain("may still be running on the warehouse") @@ -543,7 +566,6 @@ describe("snowflake savings handler over a stubbed connection", () => { return answers(sql) }) const result = await snowflakeSavings({ warehouse: "sf", now_ms: NOW }, rules) - spy.mockRestore() expect(result).toMatchObject({ success: false, error: "No check could run. not authorized" }) expect(sent).toBeUndefined() }) @@ -554,7 +576,6 @@ describe("snowflake savings handler over a stubbed connection", () => { return answers(sql) }) const result = await snowflakeSavings({ warehouse: "sf", now_ms: NOW }, rules) - spy.mockRestore() expect(result).toMatchObject({ success: false, error: "Insufficient privileges" }) expect(sent).toBeUndefined() }) @@ -562,7 +583,6 @@ describe("snowflake savings handler over a stubbed connection", () => { test("a role that can list no warehouse gets a plain failure before any usage query", async () => { const spy = connect(() => table([])) const result = await snowflakeSavings({ warehouse: "sf", now_ms: NOW }, rules) - spy.mockRestore() expect(result.success).toBe(false) expect(result.error).toContain("SHOW WAREHOUSES lists no warehouse") expect(calls.length).toBe(1) @@ -575,28 +595,54 @@ describe("snowflake savings handler over a stubbed connection", () => { : answers(sql), ) const result = await snowflakeSavings({ warehouse: "sf", now_ms: NOW }, rules) - spy.mockRestore() expect(result.skipped).toEqual([ { check: "1 warehouse that billed credits", reason: "SHOW WAREHOUSES does not list SECRET_WH: dropped, renamed, or not visible to the connection's role" }, ]) }) - test("a query past its limit fails with a plain reason, or skips an optional check", async () => { - const timeout = (cancelled: boolean) => Object.assign(new Error("QUERY_TIMEOUT: snowflake query exceeded"), { code: "QUERY_TIMEOUT", cancelled }) - let spy = connect((sql) => { - if (sql === SNOWFLAKE_METERING_FACTS_SQL) throw timeout(true) + test("usage an earlier warehouse billed under a listed name is named as not checked", async () => { + connect((sql) => + sql === SNOWFLAKE_METERING_FACTS_SQL + ? table([{ warehouse_id: 3, warehouse_name: "BI_WH", usage_day: "2026-11-11", idle_day: null, credits_used: 100 }, ...METERING]) + : answers(sql), + ) + const result = await snowflakeSavings({ warehouse: "sf", now_ms: NOW }, rules) + expect(result.warehouses_checked).toBe(1) + expect(result.skipped).toEqual([ + { + check: "Earlier usage of 1 warehouse name", + reason: "BI_WH also billed credits as an earlier warehouse, since dropped or renamed: only the current warehouse's usage was checked", + }, + ]) + }) + + const timeout = (fields: { cancelled?: boolean; unsent?: boolean }) => + Object.assign(new Error("QUERY_TIMEOUT: snowflake query exceeded"), { code: "QUERY_TIMEOUT", ...fields }) + + test("a required query past its limit fails the check with a plain reason", async () => { + connect((sql) => { + if (sql === SNOWFLAKE_METERING_FACTS_SQL) throw timeout({ cancelled: true }) return answers(sql) }) - let result = await snowflakeSavings({ warehouse: "sf", now_ms: NOW }, rules) - spy.mockRestore() + const result = await snowflakeSavings({ warehouse: "sf", now_ms: NOW }, rules) expect(result.error).toBe("The metering query ran past the check's 9-minute limit and was cancelled on the warehouse.") + }) + + test("a query stopped before it was sent says it did not run", async () => { + connect((sql) => { + if (sql === SNOWFLAKE_METERING_FACTS_SQL) throw timeout({ cancelled: false, unsent: true }) + return answers(sql) + }) + const result = await snowflakeSavings({ warehouse: "sf", now_ms: NOW }, rules) + expect(result.error).toBe("The metering query ran past the check's 9-minute limit before it was sent, so it did not run on the warehouse.") + }) - spy = connect((sql) => { - if (sql === SNOWFLAKE_QUERY_FACTS_SQL) throw timeout(false) + test("an optional query past its limit skips only its check", async () => { + connect((sql) => { + if (sql === SNOWFLAKE_QUERY_FACTS_SQL) throw timeout({ cancelled: false }) return answers(sql) }) - result = await snowflakeSavings({ warehouse: "sf", now_ms: NOW }, rules) - spy.mockRestore() + const result = await snowflakeSavings({ warehouse: "sf", now_ms: NOW }, rules) expect(result.success).toBe(true) expect(result.skipped).toEqual([ { check: "The short-query signal of Downsize Warehouse", reason: "The query history query ran past the check's 9-minute limit and may still be running on the warehouse." }, @@ -605,7 +651,7 @@ describe("snowflake savings handler over a stubbed connection", () => { test("each query gets only what the earlier ones left; with nothing left it is not started", async () => { let now = 1_000_000 - const clock = spyOn(Date, "now").mockImplementation(() => now) + const clock = tracked(spyOn(Date, "now")).mockImplementation(() => now) const spy = connect((sql) => { // The metering query uses up two minutes, and the period query all the rest. if (sql === SNOWFLAKE_METERING_FACTS_SQL) now += 120_000 @@ -613,14 +659,7 @@ describe("snowflake savings handler over a stubbed connection", () => { return answers(sql) }) const result = await snowflakeSavings({ warehouse: "sf", now_ms: NOW }, rules) - spy.mockRestore() - clock.mockRestore() - expect(calls.map((one) => one.timeout)).toEqual([ - SAVINGS_RUN_LIMIT_MS, - SAVINGS_RUN_LIMIT_MS, - SAVINGS_RUN_LIMIT_MS, - SAVINGS_RUN_LIMIT_MS - 120_000, - ]) + expect(calls.map((one) => one.timeout)).toEqual([SAVINGS_RUN_LIMIT_MS, SAVINGS_RUN_LIMIT_MS, SAVINGS_RUN_LIMIT_MS - 120_000]) expect(result.success).toBe(true) expect(result.skipped).toEqual([ { check: "Frequent Restart Overcharge", reason: "The restart events query was not started: it would run past the check's 9-minute limit." }, @@ -628,17 +667,25 @@ describe("snowflake savings handler over a stubbed connection", () => { ]) }) - test("reaching the limit before the required billing query fails the check", async () => { + test("the optional rate lookup cannot take a usage query's time: with none left, list price stands in", async () => { let now = 1_000_000 - const clock = spyOn(Date, "now").mockImplementation(() => now) - const spy = connect((sql) => { - if (sql === SNOWFLAKE_RATE_SQL) now += SAVINGS_RUN_LIMIT_MS + tracked(spyOn(Date, "now")).mockImplementation(() => now) + connect((sql) => { + // The last usage query uses up the limit. + if (sql === SNOWFLAKE_QUERY_FACTS_SQL) now += SAVINGS_RUN_LIMIT_MS return answers(sql) }) const result = await snowflakeSavings({ warehouse: "sf", now_ms: NOW }, rules) - spy.mockRestore() - clock.mockRestore() - expect(result).toMatchObject({ success: false, error: "The metering query was not started: it would run past the check's 9-minute limit." }) + expect(calls.map((one) => one.sql)).toEqual([ + SNOWFLAKE_WAREHOUSES_SQL, + SNOWFLAKE_METERING_FACTS_SQL, + SNOWFLAKE_PERIOD_FACTS_SQL, + SNOWFLAKE_RESTART_FACTS_SQL, + SNOWFLAKE_QUERY_FACTS_SQL, + ]) + expect(result.success).toBe(true) + expect(result.price).toEqual({ per_credit: LIST_PRICE_PER_CREDIT, source: "list_price" }) + expect(result.skipped).toEqual([]) }) test("stopping the session cancels the running query and starts no other", async () => { @@ -651,7 +698,6 @@ describe("snowflake savings handler over a stubbed connection", () => { return answers(sql) }) const result = await snowflakeSavings({ warehouse: "sf", now_ms: NOW }, rules, { signal: controller.signal }) - spy.mockRestore() expect(result).toMatchObject({ success: false, error: "session stopped" }) expect(result.skipped).toBeUndefined() expect(calls.at(-1)!.sql).toBe(SNOWFLAKE_PERIOD_FACTS_SQL) @@ -662,50 +708,66 @@ describe("snowflake savings handler over a stubbed connection", () => { test("connecting counts toward the limit: a connection that never comes up ends the check in time", async () => { // A 30 ms session budget stands in for the last moments of the 9 minutes. - const budget = spyOn(RunBudget, "queryDeadline").mockReturnValue(Date.now() + 30) - const get = spyOn(Registry, "get").mockReturnValue(new Promise(() => {}) as any) + const budget = tracked(spyOn(RunBudget, "queryDeadline")).mockReturnValue(Date.now() + 30) + const get = tracked(spyOn(Registry, "get")).mockReturnValue(new Promise(() => {}) as any) const result = await snowflakeSavings({ warehouse: "sf", now_ms: NOW }, rules, { sessionID: "s1" }) - get.mockRestore() - budget.mockRestore() expect(result).toMatchObject({ success: false, error: "Connecting to the warehouse took longer than the run's time budget." }) }) test("stopping the session while connecting ends the check at once", async () => { const controller = new AbortController() - const get = spyOn(Registry, "get").mockReturnValue(new Promise(() => {}) as any) + const get = tracked(spyOn(Registry, "get")).mockReturnValue(new Promise(() => {}) as any) const pending = snowflakeSavings({ warehouse: "sf", now_ms: NOW }, rules, { signal: controller.signal }) controller.abort(new Error("session stopped")) const result = await pending - get.mockRestore() expect(result).toMatchObject({ success: false, error: "session stopped" }) }) - test("a connection that fails after the session already stopped is not left unhandled", async () => { + test("a session that already stopped does not connect to Snowflake at all", async () => { const controller = new AbortController() controller.abort(new Error("session stopped")) + const get = tracked(spyOn(Registry, "get")).mockRejectedValue(new Error("connected")) + const result = await snowflakeSavings({ warehouse: "sf", now_ms: NOW }, rules, { signal: controller.signal }) + expect(result).toMatchObject({ success: false, error: "session stopped" }) + expect(get).not.toHaveBeenCalled() + }) + + test("a session time budget already used up does not connect to Snowflake at all", async () => { + tracked(spyOn(RunBudget, "queryDeadline")).mockReturnValue(Date.now() - 1) + const get = tracked(spyOn(Registry, "get")).mockRejectedValue(new Error("connected")) + const result = await snowflakeSavings({ warehouse: "sf", now_ms: NOW }, rules, { sessionID: "s1" }) + expect(result).toMatchObject({ success: false, error: "Connecting to the warehouse was not started: the run's time budget had already passed." }) + expect(get).not.toHaveBeenCalled() + }) + + test("a connection that fails after the session stopped is not left unhandled", async () => { + const controller = new AbortController() const unhandled: unknown[] = [] const note = (reason: unknown) => unhandled.push(reason) process.on("unhandledRejection", note) - let fail!: (reason: Error) => void - const connecting = new Promise((_, reject) => (fail = reject)) - const get = spyOn(Registry, "get").mockReturnValue(connecting as any) - const result = await snowflakeSavings({ warehouse: "sf", now_ms: NOW }, rules, { signal: controller.signal }) - fail(new Error("connection refused")) - await new Promise((resolve) => setTimeout(resolve, 0)) - get.mockRestore() - process.off("unhandledRejection", note) - expect(result).toMatchObject({ success: false, error: "session stopped" }) - expect(unhandled).toEqual([]) + try { + let fail!: (reason: Error) => void + const connecting = new Promise((_, reject) => (fail = reject)) + tracked(spyOn(Registry, "get")).mockReturnValue(connecting as any) + const pending = snowflakeSavings({ warehouse: "sf", now_ms: NOW }, rules, { signal: controller.signal }) + controller.abort(new Error("session stopped")) + const result = await pending + fail(new Error("connection refused")) + await new Promise((resolve) => setTimeout(resolve, 0)) + expect(result).toMatchObject({ success: false, error: "session stopped" }) + expect(unhandled).toEqual([]) + } finally { + process.off("unhandledRejection", note) + } }) test("a rate query that may still be running is named, and list price is used", async () => { - const timeout = Object.assign(new Error("QUERY_TIMEOUT"), { code: "QUERY_TIMEOUT", cancelled: false }) + const stopped = timeout({ cancelled: false }) const spy = connect((sql) => { - if (sql === SNOWFLAKE_RATE_SQL) throw timeout + if (sql === SNOWFLAKE_RATE_SQL) throw stopped return answers(sql) }) const result = await snowflakeSavings({ warehouse: "sf", now_ms: NOW }, rules) - spy.mockRestore() expect(result.price?.source).toBe("list_price") expect(result.skipped).toEqual([ { check: "The account's credit rate", reason: "The credit rate query ran past the check's 9-minute limit and may still be running on the warehouse." }, @@ -713,15 +775,13 @@ describe("snowflake savings handler over a stubbed connection", () => { }) test("a session time budget shorter than 9 minutes is the limit, and is named when it stops a query", async () => { - const budget = spyOn(RunBudget, "queryDeadline").mockReturnValue(Date.now() + 60_000) - const timeout = Object.assign(new Error("QUERY_TIMEOUT"), { code: "QUERY_TIMEOUT", cancelled: true }) + const budget = tracked(spyOn(RunBudget, "queryDeadline")).mockReturnValue(Date.now() + 60_000) + const stopped = timeout({ cancelled: true }) const spy = connect((sql) => { - if (sql === SNOWFLAKE_METERING_FACTS_SQL) throw timeout + if (sql === SNOWFLAKE_METERING_FACTS_SQL) throw stopped return answers(sql) }) const result = await snowflakeSavings({ warehouse: "sf", now_ms: NOW }, rules, { sessionID: "s1" }) - spy.mockRestore() - budget.mockRestore() for (const one of calls) expect(one.timeout).toBeLessThanOrEqual(60_000) expect(result.error).toBe("The metering query ran past the run's time budget and was cancelled on the warehouse.") }) @@ -736,7 +796,6 @@ describe("snowflake savings handler over a stubbed connection", () => { return answers(sql) }) const result = await snowflakeSavings({ warehouse: "sf", now_ms: NOW }, rules, { signal: controller.signal }) - spy.mockRestore() expect(result).toMatchObject({ success: false, error: "session stopped" }) expect(calls.at(-1)!.sql).toBe(SNOWFLAKE_RATE_SQL) }) @@ -744,7 +803,6 @@ describe("snowflake savings handler over a stubbed connection", () => { test("a query that fills the row limit is named", async () => { const spy = connect((sql) => (sql === SNOWFLAKE_PERIOD_FACTS_SQL ? table([], true) : answers(sql))) const result = await snowflakeSavings({ warehouse: "sf", now_ms: NOW }, rules) - spy.mockRestore() expect(result.limited).toEqual(["active periods"]) }) @@ -758,7 +816,6 @@ describe("snowflake savings handler over a stubbed connection", () => { test("a core without the rules is refused before any query", async () => { const spy = connect(answers) const result = await snowflakeSavings({ warehouse: "sf", now_ms: NOW }, null) - spy.mockRestore() expect(result.error).toContain("no Snowflake savings rules") expect(calls).toEqual([]) }) From 7187e47ccc16e6c08b802191150d29bfff7b44d8 Mon Sep 17 00:00:00 2001 From: Shreyas Telkar Date: Thu, 19 Nov 2026 18:14:49 -0800 Subject: [PATCH 32/34] fix: assessment mode leaves the user's workspace skills and rules in place - `syncSkills` returns before the opt-out purge in assessment mode, so the workspace skill snapshot and its `.claude/skills` copies are no longer deleted. - `workspacesOn()` reads assessment mode as `Flag` does, so rules sync is off (no lookup, no reset of pulled rules) instead of reading it as signed out. - The plugin loader's skip sits on its own install branch; the dependency skip folds into `shouldInstallDependencies` (TUI config too). - Tests: skill snapshot kept, `workspacesOn`, sign-out in assessment mode, plugin and dependency gates with controls; the zero-egress build pins the catalog fixture and the signed-in case gets a control run. Co-Authored-By: Claude Opus 5.5 --- .../data-engineering/tools/finops-tools.md | 2 +- .../opencode/src/altimate/learn/config.ts | 8 +- .../src/altimate/workspace/skill-sync.ts | 4 + packages/opencode/src/config/config.ts | 4 +- packages/opencode/src/config/plugin.ts | 5 +- packages/opencode/src/plugin/index.ts | 13 +- .../altimate/assessment-plugin-gate.test.ts | 130 ++++++++++++++++++ .../altimate/assessment-zero-egress.test.ts | 23 ++-- .../test/altimate/learn/config.test.ts | 11 +- .../opencode/test/altimate/sign-out.test.ts | 31 +++++ .../altimate/workspace/skill-sync.test.ts | 24 ++++ 11 files changed, 231 insertions(+), 24 deletions(-) create mode 100644 packages/opencode/test/altimate/assessment-plugin-gate.test.ts diff --git a/docs/docs/data-engineering/tools/finops-tools.md b/docs/docs/data-engineering/tools/finops-tools.md index 302c462a6f..f7126bbf6f 100644 --- a/docs/docs/data-engineering/tools/finops-tools.md +++ b/docs/docs/data-engineering/tools/finops-tools.md @@ -193,7 +193,7 @@ The tool measures each resource's usage; altimate-core decides what qualifies an **Asks first, and stops long queries.** The check runs only when called with `confirm: true`; without it, the agent is told to ask you first, since a run can start a stopped warehouse that then bills. It runs three queries and stops after 9 minutes, connecting included, cancelling any query still running on the warehouse (or sooner, at an `altimate run --time-budget` cutoff); stopping the session cancels the running query too. The Claude Code `/savings` command passes `confirm: true`, since typing it is the go-ahead. -**Assessment mode.** Set `ALTIMATE_ASSESSMENT_MODE=true` to keep a savings check from contacting anything but your warehouse. It sends no telemetry (not even the one-time opt-out event), treats you as signed out of Altimate (no workspace or project-binding lookups), fetches no remote config and installs no plugin dependencies, and downloads neither the model catalog (models.dev) nor ripgrep (github.com). Any value but empty, `0` or `false` turns it on, and a tool daemon started without it is not reused. It is measured for `tool run` of the two savings tools. Other tools and commands are not covered: a signed-in user's other Altimate tools can still reach Altimate, and so can the update check and language-server downloads. Core's compiled analysis is not audited for network use. Without it, `ALTIMATE_TELEMETRY_DISABLED=true` alone still lets the CLI send that opt-out event and fetch those. The Claude Code `/savings` command sets it for you. Tool results shown in a Claude Code conversation reach its model provider; assessment mode does not change that. +**Assessment mode.** Set `ALTIMATE_ASSESSMENT_MODE=true` to keep a savings check from contacting anything but your warehouse. It sends no telemetry (not even the one-time opt-out event), treats you as signed out of Altimate (no workspace or project-binding lookups; workspace skills already on disk stay as they are, and pulled team rules are not applied while it is on), fetches no remote config and installs no plugin dependencies, and downloads neither the model catalog (models.dev) nor ripgrep (github.com). Any value but empty, `0` or `false` turns it on, and a tool daemon started without it is not reused. It is measured for `tool run` of the two savings tools. Other tools and commands are not covered: a signed-in user's other Altimate tools can still reach Altimate, and so can the update check and language-server downloads. Core's compiled analysis is not audited for network use. Without it, `ALTIMATE_TELEMETRY_DISABLED=true` alone still lets the CLI send that opt-out event and fetch those. The Claude Code `/savings` command sets it for you. Tool results shown in a Claude Code conversation reach its model provider; assessment mode does not change that. ``` > finops_databricks_savings --warehouse prod_dbx --confirm diff --git a/packages/opencode/src/altimate/learn/config.ts b/packages/opencode/src/altimate/learn/config.ts index 82ece552e7..0d9554b0b9 100644 --- a/packages/opencode/src/altimate/learn/config.ts +++ b/packages/opencode/src/altimate/learn/config.ts @@ -15,9 +15,13 @@ export function booleanFlag(value: string | undefined): boolean | undefined { return undefined } -/** Workspaces are on unless the `ALTIMATE_DISABLE_WORKSPACE` kill switch is set (`Flag`'s parse, with `env` explicit). */ +/** + * Workspaces are on unless the `ALTIMATE_DISABLE_WORKSPACE` kill switch or assessment mode is set, as + * `Flag.ALTIMATE_DISABLE_WORKSPACE` reads them, with `env` explicit. Off, rules neither sync nor reset, and + * team rules already pulled are not delivered; nothing on disk is removed. + */ export function workspacesOn(env: NodeJS.ProcessEnv = process.env): boolean { - return !killSwitchSet(env["ALTIMATE_DISABLE_WORKSPACE"]) + return !killSwitchSet(env["ALTIMATE_DISABLE_WORKSPACE"]) && !killSwitchSet(env["ALTIMATE_ASSESSMENT_MODE"]) } /** The global switch defaults on; env overrides config in both directions. */ diff --git a/packages/opencode/src/altimate/workspace/skill-sync.ts b/packages/opencode/src/altimate/workspace/skill-sync.ts index efa1dd7d44..55774da784 100644 --- a/packages/opencode/src/altimate/workspace/skill-sync.ts +++ b/packages/opencode/src/altimate/workspace/skill-sync.ts @@ -1266,6 +1266,10 @@ export async function syncSkills(directory: string): Promise { // false answer waiting for the next caller to trust it. return await existing.catch(() => ({ changed: false, skipped: [] })) } + // Assessment mode only stops contact with Altimate. The snapshot and its `.claude/skills` copies + // were there before the run and stay as they are, unsynced: the off-switch purge below is for a + // user who turned workspaces off, not for a run that may not reach the server. + if (CoreFlag.ALTIMATE_ASSESSMENT_MODE) return { changed: false, skipped: [] } if (!isEnabled()) { // Opting out has to actually take effect: a snapshot left behind keeps // loading into every session. Still gated on the symlink check — this path diff --git a/packages/opencode/src/config/config.ts b/packages/opencode/src/config/config.ts index 03565568a9..0c0564fba6 100644 --- a/packages/opencode/src/config/config.ts +++ b/packages/opencode/src/config/config.ts @@ -748,9 +748,7 @@ export const layer = Layer.effect( // under any earlier directory. Keep the PURE skip and retain fibers for waitForDependencies. const deps: Fiber.Fiber[] = [] for (const dir of directories) { - // altimate_change start — assessment mode installs no plugin dependencies (npm) - if (!Flag.ALTIMATE_ASSESSMENT_MODE && ConfigPlugin.shouldInstallDependencies(dir, result.plugin)) { - // altimate_change end + if (ConfigPlugin.shouldInstallDependencies(dir, result.plugin)) { const dep = yield* npmSvc .install(dir, { add: [ diff --git a/packages/opencode/src/config/plugin.ts b/packages/opencode/src/config/plugin.ts index 7cc49f4f04..bc78cf287e 100644 --- a/packages/opencode/src/config/plugin.ts +++ b/packages/opencode/src/config/plugin.ts @@ -80,12 +80,13 @@ export function needsDependencies(dir: string, plugins: readonly ConfigPluginV1. // PURE mode skips dependency installs entirely: isolated-HOME environments and subprocess tests run // with no package cache, so the install attempt fails, npm retries, and the process hangs past exit. -// Fold that check in here so both call sites (Config and TuiConfig) stay in sync. +// Assessment mode installs nothing from npm either. Both checks live here so both call sites (Config +// and TuiConfig) stay in sync. export function shouldInstallDependencies( dir: string, plugins: readonly ConfigPluginV1.Spec[] | undefined, ): boolean { - return !Flag.OPENCODE_PURE && needsDependencies(dir, plugins) + return !Flag.OPENCODE_PURE && !Flag.ALTIMATE_ASSESSMENT_MODE && needsDependencies(dir, plugins) } // altimate_change end diff --git a/packages/opencode/src/plugin/index.ts b/packages/opencode/src/plugin/index.ts index ab68bd14cb..3e08f9e7ac 100644 --- a/packages/opencode/src/plugin/index.ts +++ b/packages/opencode/src/plugin/index.ts @@ -134,13 +134,14 @@ export namespace Plugin { // ignore old codex plugin since it is supported first party now if (plugin.includes("opencode-openai-codex-auth") || plugin.includes("opencode-copilot-auth")) continue log.info("loading plugin", { path: plugin }) - // altimate_change start — assessment mode installs no npm plugin; local file plugins still load - if (!plugin.startsWith("file://") && Flag.assessmentMode()) { - log.warn("skipping npm plugin: assessment mode installs nothing", { path: plugin }) - continue - } - // altimate_change end if (!plugin.startsWith("file://")) { + // altimate_change start — assessment mode installs nothing: this loader installs every spec but + // a `file://` one, so exactly those are skipped and `file://` plugins still load + if (Flag.assessmentMode()) { + log.warn("skipping plugin: assessment mode installs nothing", { path: plugin }) + continue + } + // altimate_change end const lastAtIndex = plugin.lastIndexOf("@") const pkg = lastAtIndex > 0 ? plugin.substring(0, lastAtIndex) : plugin const version = lastAtIndex > 0 ? plugin.substring(lastAtIndex + 1) : "latest" diff --git a/packages/opencode/test/altimate/assessment-plugin-gate.test.ts b/packages/opencode/test/altimate/assessment-plugin-gate.test.ts new file mode 100644 index 0000000000..8908cd001e --- /dev/null +++ b/packages/opencode/test/altimate/assessment-plugin-gate.test.ts @@ -0,0 +1,130 @@ +/** + * Assessment mode installs no plugin: each loader skips exactly the specs it would install, and plugins + * it loads from disk still load. + */ +import { afterEach, describe, expect, spyOn, test } from "bun:test" +import { Effect, Layer } from "effect" +import fs from "fs/promises" +import os from "os" +import path from "path" +import { pathToFileURL } from "url" +import { CrossSpawnSpawner } from "@opencode-ai/core/cross-spawn-spawner" +import { FSUtil } from "@opencode-ai/core/fs-util" +import { disposeAllInstances, provideInstance, testInstanceStoreLayer, tmpdirScoped } from "../fixture/fixture" +import { testEffect } from "../lib/effect" + +const { Plugin } = await import("../../src/plugin/index") +const { resolvePluginTarget } = await import("../../src/plugin/shared") +const { BunProc } = await import("../../src/bun") +const { Npm } = await import("@opencode-ai/core/npm") +const { EventV2Bridge } = await import("../../src/event-v2-bridge") +const { TestConfig } = await import("../fixture/config") +const { RuntimeFlags } = await import("../../src/effect/runtime-flags") +const { ConfigPlugin } = await import("../../src/config/plugin") + +const saved = process.env.ALTIMATE_ASSESSMENT_MODE +const spies: { mockRestore(): void }[] = [] + +afterEach(async () => { + for (const spy of spies.splice(0).reverse()) spy.mockRestore() + if (saved === undefined) delete process.env.ALTIMATE_ASSESSMENT_MODE + else process.env.ALTIMATE_ASSESSMENT_MODE = saved + await disposeAllInstances() +}) + +const it = testEffect(Layer.mergeAll(CrossSpawnSpawner.defaultLayer, FSUtil.defaultLayer, testInstanceStoreLayer)) + +/** Loads `plugins` through the plugin service, from a project config that lists them. */ +function load(dir: string, plugins: string[]) { + const source = path.join(dir, "opencode.json") + return Effect.gen(function* () { + yield* Effect.promise(() => Bun.write(source, JSON.stringify({ plugin: plugins }))) + const plugin = yield* Plugin.Service + yield* plugin.list() + }).pipe( + Effect.provide( + Plugin.layer.pipe( + Layer.provide(EventV2Bridge.defaultLayer), + Layer.provide(RuntimeFlags.layer({ disableDefaultPlugins: true })), + Layer.provide( + TestConfig.layer({ + get: () => + Effect.succeed({ + plugin: plugins, + plugin_origins: plugins.map((spec) => ({ spec, source, scope: "local" as const })), + }), + directories: () => Effect.succeed([dir]), + }), + ), + ), + ), + provideInstance(dir), + ) +} + +/** A file plugin that writes `mark` when it loads. */ +async function filePlugin(dir: string) { + const file = path.join(dir, "plugin.ts") + const mark = path.join(dir, "loaded.txt") + await Bun.write(file, `export default async () => {\n await Bun.write(${JSON.stringify(mark)}, "loaded")\n return {}\n}\n`) + return { spec: pathToFileURL(file).href, mark } +} + +describe("assessment mode and plugins", () => { + it.live("the plugin service installs no npm plugin, and a file:// plugin still loads", () => + Effect.gen(function* () { + const dir = yield* tmpdirScoped() + const local = yield* Effect.promise(() => filePlugin(dir)) + const install = spyOn(BunProc, "install").mockResolvedValue("") + spies.push(install) + process.env.ALTIMATE_ASSESSMENT_MODE = "1" + yield* load(dir, ["some-npm-plugin@1.0.0", local.spec]) + expect(install).not.toHaveBeenCalled() + expect(yield* Effect.promise(() => fs.readFile(local.mark, "utf8"))).toBe("loaded") + }), + ) + + it.live("control: without the mode the same npm plugin is installed", () => + Effect.gen(function* () { + const dir = yield* tmpdirScoped() + const install = spyOn(BunProc, "install").mockResolvedValue("") + spies.push(install) + delete process.env.ALTIMATE_ASSESSMENT_MODE + yield* load(dir, ["some-npm-plugin@1.0.0"]) + expect(install).toHaveBeenCalledWith("some-npm-plugin", "1.0.0") + }), + ) + + it.live("the shared resolver refuses an npm spec without installing it, and still resolves a path spec", () => + Effect.gen(function* () { + const dir = yield* tmpdirScoped() + const local = yield* Effect.promise(() => filePlugin(dir)) + const add = spyOn(Npm, "add").mockRejectedValue(new Error("installed")) + spies.push(add) + process.env.ALTIMATE_ASSESSMENT_MODE = "1" + const refused = yield* Effect.promise(() => resolvePluginTarget("some-npm-plugin@1.0.0").catch((e: Error) => e)) + expect((refused as Error).message).toBe("npm plugins are not installed in assessment mode") + expect(add).not.toHaveBeenCalled() + expect(yield* Effect.promise(() => resolvePluginTarget(local.spec))).toBe(local.spec) + }), + ) +}) + +describe("assessment mode and plugin dependencies", () => { + test("no dependency install is planned for a directory that would otherwise get one", async () => { + const dir = await fs.mkdtemp(path.join(os.tmpdir(), "assessment-deps-")) + const pure = process.env.OPENCODE_PURE + try { + await fs.mkdir(path.join(dir, "node_modules")) + delete process.env.OPENCODE_PURE + delete process.env.ALTIMATE_ASSESSMENT_MODE + expect(ConfigPlugin.shouldInstallDependencies(dir, undefined)).toBe(true) + process.env.ALTIMATE_ASSESSMENT_MODE = "1" + expect(ConfigPlugin.shouldInstallDependencies(dir, undefined)).toBe(false) + } finally { + if (pure === undefined) delete process.env.OPENCODE_PURE + else process.env.OPENCODE_PURE = pure + await fs.rm(dir, { recursive: true, force: true }) + } + }) +}) diff --git a/packages/opencode/test/altimate/assessment-zero-egress.test.ts b/packages/opencode/test/altimate/assessment-zero-egress.test.ts index f9ad659737..dfd34f10bb 100644 --- a/packages/opencode/test/altimate/assessment-zero-egress.test.ts +++ b/packages/opencode/test/altimate/assessment-zero-egress.test.ts @@ -44,7 +44,8 @@ async function packagedBinary(): Promise { cwd: root, stdout: "pipe", stderr: "pipe", - env: cleanEnv(), + // The catalog every other CI build pins, so the build does not fetch models.dev. + env: { ...cleanEnv(), MODELS_DEV_API_JSON: path.join(root, "test/tool/fixtures/models-api.json") }, }) if (build.exitCode !== 0) throw new Error(`build failed: ${build.stderr.toString()}`) const dist = path.join(root, "dist") @@ -214,16 +215,22 @@ describe("assessment mode on the packaged CLI", () => { expect(output).not.toContain("none are set up") }, 120_000) - posixOnly("signed in to Altimate: no host is contacted", async () => { - // The ambient PATH: with a bare one the project-binding lookup is never reached, so the case would prove nothing. - const result = await run(savings, { ...assessment, PATH: process.env.PATH ?? "" }, async (home) => { + posixOnly("signed in to Altimate: no host is contacted, though the same home reaches Altimate without the mode", async () => { + const backend = "altimate-backend.invalid" + const signedIn = async (home: string) => { await writeFile( path.join(home, ".altimate", "altimate.json"), - JSON.stringify({ altimateUrl: `https://${TELEMETRY_SINK}`, altimateInstanceName: "acme", altimateApiKey: "x" }), + JSON.stringify({ altimateUrl: `https://${backend}`, altimateInstanceName: "acme", altimateApiKey: "x" }), ) - }) + } + // The ambient PATH: with a bare one the project-binding lookup is never reached, so the case would prove nothing. + const ambient = { PATH: process.env.PATH ?? "" } + const result = await run(savings, { ...assessment, ...ambient }, signedIn) expect(result.hosts).toEqual([]) - }, 120_000) + // Control: the credentials are read and used, so only the mode keeps the backend lookup off. + const control = await run(savings, { ALTIMATE_TELEMETRY_DISABLED: "true", ...ambient }, signedIn) + expect(control.hosts.some((host) => host.startsWith(backend))).toBe(true) + }, 240_000) posixOnly("a well-known remote config entry and project plugin sources: no host is contacted", async () => { const result = await run(savings, assessment, async (home, project) => { @@ -242,8 +249,6 @@ describe("assessment mode on the packaged CLI", () => { }, 120_000) posixOnly("a tool daemon started without the mode does not serve a run that sets it", async () => { - const dir = await mkdtemp(path.join(tmpdir(), "assessment-daemon-")) - scratch.push(dir) const { connectionEnvKey } = await import("../../src/altimate/tool-cli/daemon") expect(connectionEnvKey({ ALTIMATE_ASSESSMENT_MODE: "true" })).not.toBe(connectionEnvKey({})) }) diff --git a/packages/opencode/test/altimate/learn/config.test.ts b/packages/opencode/test/altimate/learn/config.test.ts index ca0df7d09d..73123d0005 100644 --- a/packages/opencode/test/altimate/learn/config.test.ts +++ b/packages/opencode/test/altimate/learn/config.test.ts @@ -1,6 +1,6 @@ // altimate_change - new file import { describe, expect, test } from "bun:test" -import { autoReflectEnabled, captureEnabled, learnEnabled } from "../../../src/altimate/learn/config" +import { autoReflectEnabled, captureEnabled, learnEnabled, workspacesOn } from "../../../src/altimate/learn/config" describe("learning kill switch", () => { test("defaults enabled without configuration or an env override", () => { @@ -40,3 +40,12 @@ describe("learning kill switch", () => { expect(autoReflectEnabled(config, { ...env, ALTIMATE_LEARN: "1" })).toBe(true) }) }) + +describe("workspaces switch", () => { + test("the kill switch and assessment mode each turn workspaces off, as Flag reads them", () => { + expect(workspacesOn({})).toBe(true) + expect(workspacesOn({ ALTIMATE_DISABLE_WORKSPACE: "1" })).toBe(false) + for (const value of ["true", "1", "yes", "on"]) expect(workspacesOn({ ALTIMATE_ASSESSMENT_MODE: value })).toBe(false) + for (const value of ["", "0", "false"]) expect(workspacesOn({ ALTIMATE_ASSESSMENT_MODE: value })).toBe(true) + }) +}) diff --git a/packages/opencode/test/altimate/sign-out.test.ts b/packages/opencode/test/altimate/sign-out.test.ts index 6fb1ad4224..6747a2504b 100644 --- a/packages/opencode/test/altimate/sign-out.test.ts +++ b/packages/opencode/test/altimate/sign-out.test.ts @@ -95,6 +95,37 @@ describe("AltimateApi.signOut", () => { expect(await readJson(rootConfig)).toEqual({ mcp: { "datamate-sales": datamateEntry("gateway-test-key") } }) }) + test("in assessment mode it still removes the credentials and the MCP entries carrying the key, and sends nothing", async () => { + const saved = process.env.ALTIMATE_ASSESSMENT_MODE + const originalFetch = globalThis.fetch + let requests = 0 + globalThis.fetch = (async () => { + requests++ + throw new Error("no request expected") + }) as unknown as typeof fetch + process.env.ALTIMATE_ASSESSMENT_MODE = "1" + try { + await writeJson(AltimateApi.credentialsPath(), { + altimateUrl: "https://api.test", + altimateInstanceName: "tenant", + altimateApiKey: "gateway-test-key", + }) + await writeJson(rootConfig, { mcp: { "datamate-sales": datamateEntry("gateway-test-key") } }) + + const removed = await AltimateApi.signOut(subdir) + + expect(removed).toEqual([{ name: "datamate-sales", path: rootConfig }]) + expect(await readJson(rootConfig)).toEqual({ mcp: {} }) + expect(await fsp.stat(AltimateApi.credentialsPath()).then(() => true, () => false)).toBe(false) + expect(await AltimateApi.isConfigured()).toBe(false) + expect(requests).toBe(0) + } finally { + globalThis.fetch = originalFetch + if (saved === undefined) delete process.env.ALTIMATE_ASSESSMENT_MODE + else process.env.ALTIMATE_ASSESSMENT_MODE = saved + } + }) + test("is a no-op when there is no credential file", async () => { await writeJson(rootConfig, { mcp: { "datamate-sales": datamateEntry("gateway-test-key") } }) diff --git a/packages/opencode/test/altimate/workspace/skill-sync.test.ts b/packages/opencode/test/altimate/workspace/skill-sync.test.ts index 24a56c348c..7513e31a57 100644 --- a/packages/opencode/test/altimate/workspace/skill-sync.test.ts +++ b/packages/opencode/test/altimate/workspace/skill-sync.test.ts @@ -2549,6 +2549,30 @@ describe("workspace skill sync mirrors into .claude/skills", () => { expect(existsSync(copy("mirrored"))).toBe(false) }) + test("assessment mode leaves the snapshot and its Claude Code copies as they are, and asks the server nothing", async () => { + mkdirSync(path.join(project, ".claude"), { recursive: true }) + serve({ "pub-1": { "SKILL.md": SKILL("mirrored") } }) + await syncSkills(project) + expect(existsSync(copy("mirrored"))).toBe(true) + + let requests = 0 + globalThis.fetch = (async () => { + requests++ + throw new Error("no request expected") + }) as unknown as typeof fetch + const saved = process.env.ALTIMATE_ASSESSMENT_MODE + process.env.ALTIMATE_ASSESSMENT_MODE = "1" + try { + expect(await syncSkills(project)).toEqual({ changed: false, skipped: [] }) + } finally { + if (saved === undefined) delete process.env.ALTIMATE_ASSESSMENT_MODE + else process.env.ALTIMATE_ASSESSMENT_MODE = saved + } + expect(requests).toBe(0) + expect(existsSync(path.join(project, MANAGED))).toBe(true) + expect(existsSync(copy("mirrored"))).toBe(true) + }) + test("a project without .claude gets no copies from a sync", async () => { serve({ "pub-1": { "SKILL.md": SKILL("mirrored") } }) await syncSkills(project) From 78dc8e3f5c652c8cd80edb24f757abc19ace7913 Mon Sep 17 00:00:00 2001 From: Shreyas Telkar Date: Thu, 19 Nov 2026 18:54:43 -0800 Subject: [PATCH 33/34] fix: Databricks waits for the close on ordinary queries; savings output names what went unchecked - Every `executeQuery` call passes a signal, so ordinary Databricks queries now take the cancel path. Without `cancelAfterMs` that path awaits the statement's close again, as the plain path does; under a deadline the close still runs in the background. A failed close never fails a query that returned rows. - `finops_databricks_savings` says "No savings found in what could be checked." when a check was skipped or cut at the row limit, as the Snowflake tool does, and derives the row limit it names from `ROW_LIMIT`. - An empty warehouse inventory no longer names the connection's warehouse as if it were the scope: the check reads every workspace's warehouses. Co-Authored-By: Claude Opus 5.5 --- packages/drivers/src/databricks.ts | 8 ++++--- .../drivers/test/databricks-deadline.test.ts | 24 +++++++++++++++++++ .../native/finops/databricks-savings.ts | 7 ++++-- .../tools/finops-databricks-savings.ts | 5 ++-- .../finops-databricks-savings.test.ts | 11 ++++++++- 5 files changed, 47 insertions(+), 8 deletions(-) diff --git a/packages/drivers/src/databricks.ts b/packages/drivers/src/databricks.ts index 708211fb7b..a99d6fd8c8 100644 --- a/packages/drivers/src/databricks.ts +++ b/packages/drivers/src/databricks.ts @@ -86,9 +86,11 @@ export async function connect(config: ConnectionConfig): Promise { throw error } if (!outcome.stopped) { - // Closing frees the server's result; a slow or failed close must not hold up or fail a query - // that already returned. - void closeQuietly(operation) + // Closing frees the server's result, and a failed close must not fail a query that already + // returned. Without a deadline it is awaited as on the plain path; under one it runs in the + // background, so a slow close cannot push the query past it. + if (deadlineMs === undefined) await closeQuietly(operation) + else void closeQuietly(operation) return outcome.value } const cancelled = operation ? await cancelAndClose(operation) : false diff --git a/packages/drivers/test/databricks-deadline.test.ts b/packages/drivers/test/databricks-deadline.test.ts index a0b1c86473..0c7f0b8e4a 100644 --- a/packages/drivers/test/databricks-deadline.test.ts +++ b/packages/drivers/test/databricks-deadline.test.ts @@ -218,4 +218,28 @@ describe("Databricks cancel on abort without a deadline", () => { expect(result.rows).toEqual([[1]]) expect(cancels).toBe(0) }) + + test("with a signal and no `cancelAfterMs`, the close is awaited before the rows return, as on the plain path", async () => { + let closed = false + closeResult = async () => { + await sleep(30) + closed = true + } + const db = await connector() + await db.execute("SELECT 1", 10, [], { signal: new AbortController().signal }) + expect(closed).toBe(true) + }) + + test("under a deadline, a slow close does not hold up rows that already returned", async () => { + let closed = false + closeResult = async () => { + await sleep(200) + closed = true + } + const db = await connector() + const result = await db.execute("SELECT 1", 10, [], { cancelAfterMs: 5_000 }) + expect(result.rows).toEqual([[1]]) + expect(closed).toBe(false) + expect(closes).toBe(1) + }) }) diff --git a/packages/opencode/src/altimate/native/finops/databricks-savings.ts b/packages/opencode/src/altimate/native/finops/databricks-savings.ts index c60f5c991c..a5e4c792d4 100644 --- a/packages/opencode/src/altimate/native/finops/databricks-savings.ts +++ b/packages/opencode/src/altimate/native/finops/databricks-savings.ts @@ -26,7 +26,7 @@ const DAY_MS = 86_400_000 const WINDOW_END_MS = 6 * 3_600_000 /** Rows read per query; a query that fills it says so in the result. */ -const ROW_LIMIT = 10_000 +export const ROW_LIMIT = 10_000 export { SAVINGS_RUN_LIMIT_MS } from "./savings-run" @@ -476,7 +476,10 @@ export async function databricksSavings( } const warehouses = await read("SQL warehouses", DATABRICKS_WAREHOUSE_FACTS_SQL) if (warehouses.length === 0) { - return failed(`No SQL warehouses were found for warehouse ${runsOn} in system.compute.warehouses.`) + return failed( + `No SQL warehouses were found in system.compute.warehouses. The check reads every workspace's warehouses there; ` + + `warehouse ${runsOn} only runs the queries.`, + ) } // A table the connection may not read stops only its own check; the rest still report. const skipped: { check: string; reason: string }[] = [] diff --git a/packages/opencode/src/altimate/tools/finops-databricks-savings.ts b/packages/opencode/src/altimate/tools/finops-databricks-savings.ts index 7ec606719f..d46d43ba0b 100644 --- a/packages/opencode/src/altimate/tools/finops-databricks-savings.ts +++ b/packages/opencode/src/altimate/tools/finops-databricks-savings.ts @@ -4,6 +4,7 @@ import { Tool } from "../../tool/tool" import { Dispatcher } from "../native" import type { DatabricksSavingsFinding, DatabricksSavingsReport, DatabricksSavingsResult } from "../native/types" import { SAVINGS_RUN_LIMIT_MINUTES } from "../native/finops/savings-run" +import { ROW_LIMIT } from "../native/finops/databricks-savings" import { dollars, oneLine, range } from "./finops-formatting" /** One shape for every outcome, so a reader of the metadata never guesses which fields exist. */ @@ -85,9 +86,9 @@ export function formatSavings(result: DatabricksSavingsResult): string { ].join(", ") const lines: string[] = [`Databricks savings: ${counted} checked, the ${report.lookback_days} days to ${result.window.end}, at list price.`] for (const one of result.skipped ?? []) lines.push(`Not checked: ${one.check} (${one.reason})`) - for (const what of result.limited ?? []) lines.push(`Only the first 10,000 ${what} were read; any beyond them were not checked.`) + for (const what of result.limited ?? []) lines.push(`Only the first ${ROW_LIMIT.toLocaleString("en-US")} ${what} were read; any beyond them were not checked.`) if (report.findings.length === 0) { - lines.push("No savings found.") + lines.push(result.skipped?.length || result.limited?.length ? "No savings found in what could be checked." : "No savings found.") return lines.join("\n") } lines.push( diff --git a/packages/opencode/test/altimate/finops-databricks-savings.test.ts b/packages/opencode/test/altimate/finops-databricks-savings.test.ts index f2a823071c..7f231b115e 100644 --- a/packages/opencode/test/altimate/finops-databricks-savings.test.ts +++ b/packages/opencode/test/altimate/finops-databricks-savings.test.ts @@ -176,7 +176,16 @@ describe("finops_databricks_savings tool", () => { const tool = await initTool(FinopsDatabricksSavingsTool) const result = await tool.execute({ confirm: true }, stubCtx()) expect(result.output).toContain("5 SQL warehouses, 0 all-purpose clusters, 0 jobs checked") - expect(result.output).toContain("No savings found") + expect(result.output).toEndWith("No savings found.") + }) + + test("no findings with a check skipped or cut at the row limit says only what was checked found none", () => { + const empty = { lookback_days: 7, findings: [], total: { monthly_min: 0, monthly_max: 0, resources: 0 }, what_if: [] } + const skipped = [{ check: "Jobs on all-purpose clusters", reason: "[INSUFFICIENT_PERMISSIONS] system.lakeflow" }] + expect(formatSavings({ ...REPORT, report: empty, skipped } as any)).toEndWith("No savings found in what could be checked.") + expect(formatSavings({ ...REPORT, report: empty, limited: ["job task rows"] } as any)).toEndWith( + "No savings found in what could be checked.", + ) }) test("a failure is reported with its reason, not as no savings", async () => { From a75b3ed2cd37d548a7d72203db42d5d4d0ecb371 Mon Sep 17 00:00:00 2001 From: Shreyas Telkar Date: Thu, 19 Nov 2026 19:09:27 -0800 Subject: [PATCH 34/34] fix: a client-side query timeout abandons the statement; only a stop cancels it - `executeQuery` aborts its signal with a `QueryTimeoutError` when its own deadline passes, and that error says the statement may still be running. The drivers now treat that abort as an abandon, as before this branch, so a long write past the timeout still finishes. Any other abort (a stopped session) cancels the statement on the warehouse; `cancelAfterMs` still sets a server-side deadline. - Snowflake does not reopen a closed session for a statement that was already stopped. - The Snowflake check names a rate lookup the time limit stopped, instead of reporting the rate sheet as unreadable. - Workspace-off messages name assessment mode, which also turns workspaces off. - The Databricks tool says it picks the first connection whose http_path names a SQL warehouse. Co-Authored-By: Claude Opus 5.5 --- packages/drivers/src/databricks.ts | 9 ++--- packages/drivers/src/snowflake.ts | 17 +++++++--- packages/drivers/src/stop.ts | 26 +++++++++++++-- packages/drivers/src/types.ts | 7 ++-- .../drivers/test/databricks-deadline.test.ts | 18 +++++++++- .../drivers/test/snowflake-deadline.test.ts | 33 +++++++++++++++++-- packages/drivers/test/stop.test.ts | 21 +++++++++++- .../src/altimate/native/finops/savings-run.ts | 11 ++++--- .../native/finops/snowflake-savings.ts | 7 ++-- .../tools/finops-databricks-savings.ts | 4 ++- .../opencode/src/cli/cmd/workspace-pilot.ts | 4 +-- packages/opencode/src/server/server.ts | 2 +- .../altimate/finops-snowflake-savings.test.ts | 7 ++-- .../server/altimate-workspace-routes.test.ts | 2 +- 14 files changed, 135 insertions(+), 33 deletions(-) diff --git a/packages/drivers/src/databricks.ts b/packages/drivers/src/databricks.ts index a99d6fd8c8..f6ae7952b9 100644 --- a/packages/drivers/src/databricks.ts +++ b/packages/drivers/src/databricks.ts @@ -4,7 +4,7 @@ import type { ConnectionConfig, Connector, ConnectorResult, ExecuteOptions, SchemaColumn } from "./types" import { QueryTimeoutError } from "./types" -import { untilStopped, type Stopped } from "./stop" +import { isStopRequest, untilStopped, type Stopped } from "./stop" import { loadOptionalDriver } from "./resolve" export async function connect(config: ConnectionConfig): Promise { @@ -41,8 +41,9 @@ export async function connect(config: ConnectionConfig): Promise { return cancelled } - // Past `cancelAfterMs`, or when `signal` aborts, the statement is cancelled on the warehouse: SQL - // warehouses ignore the client's server-side `queryTimeout`. With neither, it runs to the end. + // Past `cancelAfterMs`, or when `signal` is stopped, the statement is cancelled on the warehouse: SQL + // warehouses ignore the client's server-side `queryTimeout`. Otherwise, a client-side query timeout + // included, it runs to the end. async function runStatement( query: string, stmtOptions: Record, @@ -79,7 +80,7 @@ export async function connect(config: ConnectionConfig): Promise { })() let outcome: Stopped try { - outcome = await untilStopped(work, { deadlineMs, signal, onStop: () => (stopped = true) }) + outcome = await untilStopped(work, { deadlineMs, signal, stopsOnAbort: isStopRequest, onStop: () => (stopped = true) }) } catch (error) { // The fetch failed before any stop: the statement is still open on the session. if (operation) void closeQuietly(operation) diff --git a/packages/drivers/src/snowflake.ts b/packages/drivers/src/snowflake.ts index efc7302e58..a9a3bac14f 100644 --- a/packages/drivers/src/snowflake.ts +++ b/packages/drivers/src/snowflake.ts @@ -6,7 +6,7 @@ import * as fs from "fs" import type { ConnectionConfig, Connector, ConnectorResult, ExecuteOptions, SchemaColumn } from "./types" import { toPkcs8Pem } from "./snowflake-key" import { QueryTimeoutError } from "./types" -import { untilStopped } from "./stop" +import { isStopRequest, untilStopped } from "./stop" import { loadOptionalDriver } from "./resolve" import { emitBrowserSignIn, openInBrowser } from "./sign-in" import { emitReconnect, type ReconnectEvent } from "./reconnect-events" @@ -509,7 +509,9 @@ export async function connect( await noteSession(sql, binds, ranIn, startedInEpoch) return result } catch (err) { - if (!connectOptions || !isClosedConnectionError(err)) throw err + // A stopped statement is not run again, so its closed session is not reopened for it (a reopen can + // mean a browser sign-in): the next statement reopens it if it is still down. + if (!connectOptions || !isClosedConnectionError(err) || stop?.stopped) throw err // Reopen only if the failed connection is still the current one: a late error from a connection // another statement already replaced must not tear down its replacement. if (connection === used) await reconnect("closed-during-statement") @@ -597,8 +599,8 @@ export async function connect( } /** - * Past `cancelAfterMs`, or when `signal` aborts, the statement is cancelled on Snowflake. One still waiting on - * a reconnect is never sent, so nothing is left running. + * Past `cancelAfterMs`, or when `signal` is stopped, the statement is cancelled on Snowflake. One still waiting + * on a reconnect is never sent, so nothing is left running. A client-side query timeout cancels nothing. */ async function executeWithCancel( sql: string, @@ -609,7 +611,12 @@ export async function connect( signal?.throwIfAborted() const stop: StatementStop = { statements: [] } const work = executeQuery(sql, binds, stop) - const outcome = await untilStopped(work, { deadlineMs, signal, onStop: (why) => (stop.stopped = why) }) + const outcome = await untilStopped(work, { + deadlineMs, + signal, + stopsOnAbort: isStopRequest, + onStop: (why) => (stop.stopped = why), + }) if (!outcome.stopped) return outcome.value // A statement still waiting on a reconnect was never sent: nothing ran, so nothing was cancelled. const unsent = stop.statements.length === 0 diff --git a/packages/drivers/src/stop.ts b/packages/drivers/src/stop.ts index c5913336d7..0f44532884 100644 --- a/packages/drivers/src/stop.ts +++ b/packages/drivers/src/stop.ts @@ -4,19 +4,36 @@ */ import { MAX_TIMER_MS } from "./query-timeout" +import { isQueryTimeoutError } from "./types" export type StopReason = "deadline" | "aborted" export type Stopped = { stopped?: undefined; value: T } | { stopped: StopReason } +/** + * Whether an abort asks the drivers to cancel the statement. `executeQuery` aborts its signal with a + * `QueryTimeoutError` when its own client-side deadline passes: that deadline abandons the statement + * (its error says it may still be running), and a caller that wants one enforced on the server sets + * `cancelAfterMs`. Any other abort, such as a stopped session, is a stop. + */ +export function isStopRequest(reason: unknown): boolean { + return !isQueryTimeoutError(reason) +} + /** * Waits for `work`, or until `deadlineMs` passes or `signal` aborts, whichever comes first. `onStop` runs * at the moment of the stop, before this returns, so work still in flight can see it. A stopped - * `work` that fails later does not go unhandled. No deadline waits on the signal alone. + * `work` that fails later does not go unhandled. No deadline waits on the signal alone. An abort that + * `stopsOnAbort` returns false for is not a stop: `work` is then waited on to its end. */ export async function untilStopped( work: Promise, - opts: { deadlineMs?: number; signal?: AbortSignal; onStop?: (why: StopReason) => void }, + opts: { + deadlineMs?: number + signal?: AbortSignal + onStop?: (why: StopReason) => void + stopsOnAbort?: (reason: unknown) => boolean + }, ): Promise> { const { deadlineMs, signal } = opts let stopped: StopReason | undefined @@ -29,7 +46,10 @@ export async function untilStopped( wake() } const timer = deadlineMs === undefined ? undefined : setTimeout(() => halt("deadline"), Math.min(Math.max(0, deadlineMs), MAX_TIMER_MS)) - const onAbort = () => halt("aborted") + const onAbort = () => { + if (opts.stopsOnAbort?.(signal!.reason) === false) return + halt("aborted") + } if (signal?.aborted) onAbort() else signal?.addEventListener("abort", onAbort, { once: true }) try { diff --git a/packages/drivers/src/types.ts b/packages/drivers/src/types.ts index e50c0c3f58..a2f9b63f44 100644 --- a/packages/drivers/src/types.ts +++ b/packages/drivers/src/types.ts @@ -45,9 +45,10 @@ export interface ExecuteOptions { * 120 s default (see `query-timeout.ts`). */ queryTimeoutMs?: number /** Cancel the statement on the server once this many milliseconds pass. Opt-in: - * `executeQuery` fills in `queryTimeoutMs` for every call, which cancels nothing, so - * only callers that set this get a server-side deadline. An aborted `signal` cancels - * the statement with or without it. Databricks and Snowflake. */ + * `executeQuery` fills in `queryTimeoutMs` for every call, and when that deadline passes + * it aborts `signal` with a `QueryTimeoutError`, which abandons the statement but cancels + * nothing. So only callers that set this get a server-side deadline. Any other abort of + * `signal` cancels the statement with or without it. Databricks and Snowflake. */ cancelAfterMs?: number /** DuckDB only: called once after handing this query to its helper; callback errors do not affect execution. */ onDispatch?: () => void diff --git a/packages/drivers/test/databricks-deadline.test.ts b/packages/drivers/test/databricks-deadline.test.ts index 0c7f0b8e4a..c6e89d7842 100644 --- a/packages/drivers/test/databricks-deadline.test.ts +++ b/packages/drivers/test/databricks-deadline.test.ts @@ -5,7 +5,7 @@ */ import { beforeEach, describe, expect, mock, test } from "bun:test" import { restoreModuleMocksAfterTests } from "./module-mock" -import { isQueryTimeoutError } from "../src/types" +import { isQueryTimeoutError, QueryTimeoutError } from "../src/types" await restoreModuleMocksAfterTests("@databricks/sql") @@ -219,6 +219,22 @@ describe("Databricks cancel on abort without a deadline", () => { expect(cancels).toBe(0) }) + test("a client-side query timeout abandons the statement: it is not cancelled and runs to its end", async () => { + fetch = async () => { + await sleep(40) + return [{ n: 1 }] + } + const controller = new AbortController() + const db = await connector() + const pending = db.execute("SELECT 1", 10, [], { queryTimeoutMs: 10, signal: controller.signal }) + await sleep(10) + // As `executeQuery` aborts its signal when its own deadline passes. + controller.abort(new QueryTimeoutError({ engine: "databricks", timeoutMs: 10, cancelled: false })) + const result = await pending + expect(result.rows).toEqual([[1]]) + expect([cancels, closes]).toEqual([0, 1]) + }) + test("with a signal and no `cancelAfterMs`, the close is awaited before the rows return, as on the plain path", async () => { let closed = false closeResult = async () => { diff --git a/packages/drivers/test/snowflake-deadline.test.ts b/packages/drivers/test/snowflake-deadline.test.ts index c807cc9d5b..00dea3951d 100644 --- a/packages/drivers/test/snowflake-deadline.test.ts +++ b/packages/drivers/test/snowflake-deadline.test.ts @@ -4,7 +4,7 @@ */ import { beforeEach, describe, expect, test } from "bun:test" import { connect, keepSdkOffCloudMetadata } from "../src/snowflake" -import { isQueryTimeoutError } from "../src/types" +import { isQueryTimeoutError, QueryTimeoutError } from "../src/types" const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)) @@ -17,6 +17,8 @@ let sent: string[] = [] let up = true let holdConnect: Promise | undefined let failFirstWith: { code: string; message: string } | undefined +let failFirstAfterMs = 1 +let connections = 0 beforeEach(() => { finishAfterMs = undefined @@ -28,12 +30,15 @@ beforeEach(() => { up = true holdConnect = undefined failFirstWith = undefined + failFirstAfterMs = 1 + connections = 0 }) function fakeSdk() { return { configure() {}, createConnection() { + connections++ return { connect(cb: (err: Error | null) => void) { if (holdConnect) holdConnect.then(() => cb(null)) @@ -45,7 +50,7 @@ function fakeSdk() { if (failFirstWith) { const failure = Object.assign(new Error(failFirstWith.message), { code: failFirstWith.code }) failFirstWith = undefined - setTimeout(() => options.complete(failure, null, []), 1) + setTimeout(() => options.complete(failure, null, []), failFirstAfterMs) } else if (queryError) setTimeout(() => options.complete(queryError!, null, []), 1) else if (finishAfterMs !== undefined) { setTimeout(() => options.complete(null, null, [{ N: 1 }]), finishAfterMs) @@ -154,6 +159,30 @@ describe("Snowflake server-side cancel", () => { expect(cancels).toBe(0) }) + test("a client-side query timeout abandons the statement: it is not cancelled and runs to its end", async () => { + finishAfterMs = 40 + const controller = new AbortController() + const db = await connector() + const pending = db.execute("SELECT 1", 10, [], { queryTimeoutMs: 10, signal: controller.signal }) + await sleep(10) + // As `executeQuery` aborts its signal when its own deadline passes. + controller.abort(new QueryTimeoutError({ engine: "snowflake", timeoutMs: 10, cancelled: false })) + const result = await pending + expect(result.rows).toEqual([[1]]) + expect(cancels).toBe(0) + }) + + test("a session that closes under a statement already stopped is not reopened for it", async () => { + failFirstWith = { code: "407002", message: "Unable to perform operation using terminated connection." } + failFirstAfterMs = 40 + const db = await connector() + const error = await db.execute("SELECT 1", 10, [], { cancelAfterMs: 15 }).catch((e) => e) + expect(isQueryTimeoutError(error)).toBe(true) + await sleep(60) + expect(connections).toBe(1) + expect(sent).toEqual(["SELECT 1 LIMIT 11"]) + }) + test("an already-aborted signal sends nothing", async () => { const controller = new AbortController() controller.abort(new Error("session stopped")) diff --git a/packages/drivers/test/stop.test.ts b/packages/drivers/test/stop.test.ts index c29ff19515..cef3679381 100644 --- a/packages/drivers/test/stop.test.ts +++ b/packages/drivers/test/stop.test.ts @@ -1,5 +1,6 @@ import { describe, expect, test } from "bun:test" -import { untilStopped } from "../src/stop" +import { isStopRequest, untilStopped } from "../src/stop" +import { QueryTimeoutError } from "../src/types" const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)) const never = () => new Promise(() => {}) @@ -33,6 +34,24 @@ describe("untilStopped", () => { expect(await untilStopped(never(), { deadlineMs: 1_000, signal: controller.signal })).toEqual({ stopped: "aborted" }) }) + test("an abort that stopsOnAbort declines is not a stop: the work is waited on to its end", async () => { + const controller = new AbortController() + const stops: string[] = [] + const work = sleep(30).then(() => 7) + const pending = untilStopped(work, { signal: controller.signal, stopsOnAbort: () => false, onStop: (why) => stops.push(why) }) + await sleep(5) + controller.abort(new Error("ignored")) + expect(await pending).toEqual({ value: 7 }) + expect(stops).toEqual([]) + }) + + test("a client-side query timeout is not a stop request; any other abort is", () => { + // As `executeQuery` builds it when its own deadline passes. + expect(isStopRequest(new QueryTimeoutError({ engine: "snowflake", timeoutMs: 120_000, cancelled: false }))).toBe(false) + expect(isStopRequest(new Error("session stopped"))).toBe(true) + expect(isStopRequest(new DOMException("The operation was aborted.", "AbortError"))).toBe(true) + }) + test("work that fails after the stop does not go unhandled", async () => { const unhandled: unknown[] = [] const onUnhandled = (error: unknown) => unhandled.push(error) diff --git a/packages/opencode/src/altimate/native/finops/savings-run.ts b/packages/opencode/src/altimate/native/finops/savings-run.ts index 30fc6798e8..928daa5225 100644 --- a/packages/opencode/src/altimate/native/finops/savings-run.ts +++ b/packages/opencode/src/altimate/native/finops/savings-run.ts @@ -17,6 +17,9 @@ export const SAVINGS_RUN_LIMIT_MS = 9 * 60_000 /** The limit as the tools and their messages state it. */ export const SAVINGS_RUN_LIMIT_MINUTES = SAVINGS_RUN_LIMIT_MS / 60_000 +/** Work the check's time limit stopped, or never started. */ +export class SavingsLimitError extends Error {} + /** * One check's run: it stops at its own limit, or earlier at the session's query cutoff (`altimate run * --time-budget`). Stopping the session cancels the running query and starts no other. @@ -35,10 +38,10 @@ export function savingsRun(context?: QueryContext) { async function beforeStop(start: () => Promise, what: string): Promise { signal?.throwIfAborted() const left = stopsAt - Date.now() - if (left <= 0) throw new Error(`${what} was not started: ${limitName} had already passed.`) + if (left <= 0) throw new SavingsLimitError(`${what} was not started: ${limitName} had already passed.`) const outcome = await untilStopped(start(), { deadlineMs: left, signal }) if (outcome.stopped === "aborted") throw signal!.reason - if (outcome.stopped) throw new Error(`${what} took longer than ${limitName}.`) + if (outcome.stopped) throw new SavingsLimitError(`${what} took longer than ${limitName}.`) return outcome.value } @@ -46,7 +49,7 @@ export function savingsRun(context?: QueryContext) { async function query(connector: Connector, what: string, sql: string, rowLimit: number, binds: unknown[]) { signal?.throwIfAborted() const left = stopsAt - Date.now() - if (left <= 0) throw new Error(`The ${what} query was not started: it would run past ${limitName}.`) + if (left <= 0) throw new SavingsLimitError(`The ${what} query was not started: it would run past ${limitName}.`) try { return await connector.execute(sql, rowLimit, binds, { cancelAfterMs: left, signal }) } catch (e) { @@ -57,7 +60,7 @@ export function savingsRun(context?: QueryContext) { : cancelled ? "and was cancelled on the warehouse" : "and may still be running on the warehouse" - throw new Error(`The ${what} query ran past ${limitName} ${outcome}.`) + throw new SavingsLimitError(`The ${what} query ran past ${limitName} ${outcome}.`) } } diff --git a/packages/opencode/src/altimate/native/finops/snowflake-savings.ts b/packages/opencode/src/altimate/native/finops/snowflake-savings.ts index 157de716a0..6020e9a089 100644 --- a/packages/opencode/src/altimate/native/finops/snowflake-savings.ts +++ b/packages/opencode/src/altimate/native/finops/snowflake-savings.ts @@ -16,7 +16,7 @@ import * as Registry from "../connections/registry" import { silenceWith, type CoreTelemetrySwitch } from "../core-telemetry" import type { QueryContext } from "../query-execution" import { savingsWindow } from "./databricks-savings" -import { reasonOf, savingsRun } from "./savings-run" +import { reasonOf, SavingsLimitError, savingsRun } from "./savings-run" import { resolveFinopsWarehouse } from "./warehouse-resolver" import type { SnowflakeSavingsParams, SnowflakeSavingsReport, SnowflakeSavingsResult } from "../types" @@ -498,8 +498,9 @@ export async function snowflakeSavings( // query its time. Most accounts cannot read it (it needs organization access); list price stands in. const rate = await read("credit rate", SNOWFLAKE_RATE_SQL).catch((e) => { if (signal?.aborted) throw e - // An unreadable rate sheet is ordinary; a query that may still be running is worth naming. - if (reasonOf(e).includes("may still be running")) skipped.push({ check: "The account's credit rate", reason: reasonOf(e) }) + // An unreadable rate sheet is ordinary; a lookup the time limit stopped, or that may still be + // running, is named, so the list price is not passed off as an unreadable rate sheet. + if (e instanceof SavingsLimitError) skipped.push({ check: "The account's credit rate", reason: reasonOf(e) }) return [] as Row[] }) const price = creditPrice(rate[0]) diff --git a/packages/opencode/src/altimate/tools/finops-databricks-savings.ts b/packages/opencode/src/altimate/tools/finops-databricks-savings.ts index d46d43ba0b..956fe60bd9 100644 --- a/packages/opencode/src/altimate/tools/finops-databricks-savings.ts +++ b/packages/opencode/src/altimate/tools/finops-databricks-savings.ts @@ -118,7 +118,9 @@ export const FinopsDatabricksSavingsTool = Tool.define("finops_databricks_saving warehouse: z .string() .optional() - .describe("Databricks connection name. Optional — omit to use the first configured Databricks connection."), + .describe( + "Databricks connection name. Optional — omit to use the first configured Databricks connection whose http_path names a SQL warehouse.", + ), confirm: z .boolean() .optional() diff --git a/packages/opencode/src/cli/cmd/workspace-pilot.ts b/packages/opencode/src/cli/cmd/workspace-pilot.ts index f402ed9384..31f4691698 100644 --- a/packages/opencode/src/cli/cmd/workspace-pilot.ts +++ b/packages/opencode/src/cli/cmd/workspace-pilot.ts @@ -2,11 +2,11 @@ import { cmd } from "./cmd" import { UI } from "../ui" -/** What a workspace command says when `ALTIMATE_DISABLE_WORKSPACE` is set. Registered hidden in its +/** What a workspace command says when `ALTIMATE_DISABLE_WORKSPACE` or assessment mode is set. Registered hidden in its * place: unregistered, `altimate-code link` fell through to the default command, which read * "link" as a project directory and failed with "Failed to change directory to …/link". */ export const WORKSPACE_PILOT_OFF_MESSAGE = - "Workspaces are turned off because ALTIMATE_DISABLE_WORKSPACE is set. Unset it to use this command." + "Workspaces are turned off because ALTIMATE_DISABLE_WORKSPACE or ALTIMATE_ASSESSMENT_MODE is set. Unset it to use this command." export function pilotOffCommand(command: string) { return cmd({ diff --git a/packages/opencode/src/server/server.ts b/packages/opencode/src/server/server.ts index 283c322e07..953da4199f 100644 --- a/packages/opencode/src/server/server.ts +++ b/packages/opencode/src/server/server.ts @@ -96,7 +96,7 @@ export namespace Server { status: 409, body: { ok: false, - error: "Workspaces are turned off on this server because ALTIMATE_DISABLE_WORKSPACE is set.", + error: "Workspaces are turned off on this server because ALTIMATE_DISABLE_WORKSPACE or ALTIMATE_ASSESSMENT_MODE is set.", }, } } diff --git a/packages/opencode/test/altimate/finops-snowflake-savings.test.ts b/packages/opencode/test/altimate/finops-snowflake-savings.test.ts index 7ff46835f1..77d4131be8 100644 --- a/packages/opencode/test/altimate/finops-snowflake-savings.test.ts +++ b/packages/opencode/test/altimate/finops-snowflake-savings.test.ts @@ -664,10 +664,11 @@ describe("snowflake savings handler over a stubbed connection", () => { expect(result.skipped).toEqual([ { check: "Frequent Restart Overcharge", reason: "The restart events query was not started: it would run past the check's 9-minute limit." }, { check: "The short-query signal of Downsize Warehouse", reason: "The query history query was not started: it would run past the check's 9-minute limit." }, + { check: "The account's credit rate", reason: "The credit rate query was not started: it would run past the check's 9-minute limit." }, ]) }) - test("the optional rate lookup cannot take a usage query's time: with none left, list price stands in", async () => { + test("the optional rate lookup cannot take a usage query's time: with none left, list price stands in and the result says why", async () => { let now = 1_000_000 tracked(spyOn(Date, "now")).mockImplementation(() => now) connect((sql) => { @@ -685,7 +686,9 @@ describe("snowflake savings handler over a stubbed connection", () => { ]) expect(result.success).toBe(true) expect(result.price).toEqual({ per_credit: LIST_PRICE_PER_CREDIT, source: "list_price" }) - expect(result.skipped).toEqual([]) + expect(result.skipped).toEqual([ + { check: "The account's credit rate", reason: "The credit rate query was not started: it would run past the check's 9-minute limit." }, + ]) }) test("stopping the session cancels the running query and starts no other", async () => { diff --git a/packages/opencode/test/server/altimate-workspace-routes.test.ts b/packages/opencode/test/server/altimate-workspace-routes.test.ts index 3a3def5401..fcdad42859 100644 --- a/packages/opencode/test/server/altimate-workspace-routes.test.ts +++ b/packages/opencode/test/server/altimate-workspace-routes.test.ts @@ -137,7 +137,7 @@ describe("POST /altimate/workspace/refresh", () => { // The refusal names the switch, so an operator knows what to unset. expect(await response.json()).toEqual({ ok: false, - error: "Workspaces are turned off on this server because ALTIMATE_DISABLE_WORKSPACE is set.", + error: "Workspaces are turned off on this server because ALTIMATE_DISABLE_WORKSPACE or ALTIMATE_ASSESSMENT_MODE is set.", }) expect(refresh).not.toHaveBeenCalled() })