From b7cfc0426d60963a83f22a8f5c6053c7b7fce8e4 Mon Sep 17 00:00:00 2001 From: Harish Kumar Date: Fri, 13 Nov 2026 16:30:43 +0530 Subject: [PATCH 01/26] feat: foundation for the dbt Power User tools Nine tool stubs exposed through `altimate tool run` (dbt_run_results, dbt_defer_state, dbt_map_relations, docs_share_comments, dbt_cost_estimate, dbt_health, dbt_sql_graph, dbt_docs_coverage, dbt_cte_profile), an exported `AltimateApi.apiRequest` that calls the backend with the stored credentials, and `dbtAdapterFor`, the one gate every adapter-backed tool goes through: it refuses read-only calls, other projects, and an adapter that did not start. Co-Authored-By: Claude Opus 5.5 --- packages/opencode/src/altimate/api/client.ts | 18 ++ .../src/altimate/tool-cli/allow-list.ts | 18 ++ .../src/altimate/tools/dbt-adapter-access.ts | 100 ++++++++ .../src/altimate/tools/dbt-cost-estimate.ts | 17 ++ .../src/altimate/tools/dbt-cte-profile.ts | 17 ++ .../src/altimate/tools/dbt-defer-state.ts | 16 ++ .../src/altimate/tools/dbt-docs-coverage.ts | 17 ++ .../opencode/src/altimate/tools/dbt-health.ts | 16 ++ .../src/altimate/tools/dbt-map-relations.ts | 18 ++ .../src/altimate/tools/dbt-run-results.ts | 16 ++ .../src/altimate/tools/dbt-sql-graph.ts | 16 ++ .../src/altimate/tools/docs-share-comments.ts | 20 ++ .../test/altimate/dbt-adapter-access.test.ts | 221 ++++++++++++++++++ 13 files changed, 510 insertions(+) create mode 100644 packages/opencode/src/altimate/tools/dbt-adapter-access.ts create mode 100644 packages/opencode/src/altimate/tools/dbt-cost-estimate.ts create mode 100644 packages/opencode/src/altimate/tools/dbt-cte-profile.ts create mode 100644 packages/opencode/src/altimate/tools/dbt-defer-state.ts create mode 100644 packages/opencode/src/altimate/tools/dbt-docs-coverage.ts create mode 100644 packages/opencode/src/altimate/tools/dbt-health.ts create mode 100644 packages/opencode/src/altimate/tools/dbt-map-relations.ts create mode 100644 packages/opencode/src/altimate/tools/dbt-run-results.ts create mode 100644 packages/opencode/src/altimate/tools/dbt-sql-graph.ts create mode 100644 packages/opencode/src/altimate/tools/docs-share-comments.ts create mode 100644 packages/opencode/test/altimate/dbt-adapter-access.test.ts diff --git a/packages/opencode/src/altimate/api/client.ts b/packages/opencode/src/altimate/api/client.ts index 7a03aeb906..069a182b7a 100644 --- a/packages/opencode/src/altimate/api/client.ts +++ b/packages/opencode/src/altimate/api/client.ts @@ -347,6 +347,24 @@ export namespace AltimateApi { // altimate_change end } + /** + * Calls an Altimate API endpoint with the stored credentials (Bearer key and x-tenant headers) and + * returns the parsed JSON body. Endpoint functions in other files go through this. Fails with a + * message naming the credentials file when none are stored or they can't be read. + */ + export async function apiRequest(method: string, endpoint: string, body?: unknown): Promise { + let creds: AltimateCredentials + try { + creds = await getCredentials() + } catch (e) { + const p = credentialsPath() + if (!(await Filesystem.exists(p))) + throw new Error(`No Altimate credentials are stored: sign in to Altimate in altimate-code first (they are saved to ${p}).`) + throw new Error(`The Altimate credentials in ${p} can't be used: ${e instanceof Error ? e.message : String(e)}`) + } + return request(creds, method, endpoint, body) + } + export async function listDatamates(creds?: AltimateCredentials) { creds ??= await getCredentials() const data = await request(creds, "GET", "/datamates/") diff --git a/packages/opencode/src/altimate/tool-cli/allow-list.ts b/packages/opencode/src/altimate/tool-cli/allow-list.ts index 33b452d75c..0bb4f18473 100644 --- a/packages/opencode/src/altimate/tool-cli/allow-list.ts +++ b/packages/opencode/src/altimate/tool-cli/allow-list.ts @@ -35,10 +35,19 @@ import { AltimateCoreTestgenTool } from "../tools/altimate-core-testgen" import { AltimateCoreTrackLineageTool } from "../tools/altimate-core-track-lineage" import { AltimateCoreValidateTool } from "../tools/altimate-core-validate" import { DataDiffTool } from "../tools/data-diff" +import { DbtCostEstimateTool } from "../tools/dbt-cost-estimate" +import { DbtCteProfileTool } from "../tools/dbt-cte-profile" +import { DbtDeferStateTool } from "../tools/dbt-defer-state" +import { DbtDocsCoverageTool } from "../tools/dbt-docs-coverage" +import { DbtHealthTool } from "../tools/dbt-health" import { DbtLineageTool } from "../tools/dbt-lineage" import { DbtManifestTool } from "../tools/dbt-manifest" +import { DbtMapRelationsTool } from "../tools/dbt-map-relations" import { DbtProfilesTool } from "../tools/dbt-profiles" +import { DbtRunResultsTool } from "../tools/dbt-run-results" +import { DbtSqlGraphTool } from "../tools/dbt-sql-graph" import { DbtUnitTestGenTool } from "../tools/dbt-unit-test-gen" +import { DocsShareCommentsTool } from "../tools/docs-share-comments" import { FinopsAnalyzeCreditsTool } from "../tools/finops-analyze-credits" import { FinopsExpensiveQueriesTool } from "../tools/finops-expensive-queries" import { FinopsQueryHistoryTool } from "../tools/finops-query-history" @@ -106,10 +115,19 @@ export const EXPOSED: Record = { altimate_core_track_lineage: AltimateCoreTrackLineageTool, altimate_core_validate: AltimateCoreValidateTool, data_diff: DataDiffTool, + dbt_cost_estimate: DbtCostEstimateTool, + dbt_cte_profile: DbtCteProfileTool, + dbt_defer_state: DbtDeferStateTool, + dbt_docs_coverage: DbtDocsCoverageTool, + dbt_health: DbtHealthTool, dbt_lineage: DbtLineageTool, dbt_manifest: DbtManifestTool, + dbt_map_relations: DbtMapRelationsTool, dbt_profiles: DbtProfilesTool, + dbt_run_results: DbtRunResultsTool, + dbt_sql_graph: DbtSqlGraphTool, dbt_unit_test_gen: DbtUnitTestGenTool, + docs_share_comments: DocsShareCommentsTool, finops_analyze_credits: FinopsAnalyzeCreditsTool, finops_expensive_queries: FinopsExpensiveQueriesTool, finops_query_history: FinopsQueryHistoryTool, diff --git a/packages/opencode/src/altimate/tools/dbt-adapter-access.ts b/packages/opencode/src/altimate/tools/dbt-adapter-access.ts new file mode 100644 index 0000000000..a2711e35d1 --- /dev/null +++ b/packages/opencode/src/altimate/tools/dbt-adapter-access.ts @@ -0,0 +1,100 @@ +import fs from "fs" +import path from "path" +import type { Tool } from "../../tool/tool" +import { projectRoot } from "../native/connections/registry" + +/** Why a tool may not use the dbt adapter. */ +export type DbtAdapterRefusal = "read_only" | "other_project" | "no_project" | "not_started" | "not_loaded" + +/** The project's dbt adapter, or why this call may not use it (`reason` is a sentence for the user). */ +export type DbtAdapterAccess = + | { adapter: any; refusal?: undefined; reason?: undefined } + | { adapter?: undefined; refusal: DbtAdapterRefusal; reason: string } + +/** Test seams. */ +export const dbtAdapterAccessInternals = { + /** How long a call waits for the dbt adapter to start before it gives up; the start goes on for the next call. */ + adapterStartTimeoutMs: 120_000, +} + +/** + * The dbt adapter for a tool call, which every adapter-backed tool goes through. Starting the adapter + * runs dbt deps and writes target/, and compiling renders Jinja, which can run SQL: a read-only call + * (no write permission, or plan mode) never starts or uses it. Nor does a session in a dbt project the + * adapter isn't set up for. The wait for the adapter to start is bounded and follows the call's abort. + */ +export async function dbtAdapterFor(ctx: Pick): Promise { + const signal = ctx.abort + if (ctx.extra?.forceReadOnly === true || ctx.agent === "plan") + return { + refusal: "read_only", + reason: "this needs the dbt adapter, which a read-only call doesn't start: re-run with --allow-write", + } + try { + const { getDbtAdapter, getDbtProjectRoot } = await import("../native/connections/register") + // The import yields: a call cancelled meanwhile starts nothing. + signal.throwIfAborted() + const project = await sessionProject() + signal.throwIfAborted() + if (project?.other) { + return { + refusal: "other_project", + reason: project.root + ? `the dbt adapter is set up for ${project.root}, not ${project.other}` + : `the dbt adapter isn't set up for ${project.other}`, + } + } + let stop = () => {} + let timer: ReturnType | undefined + const waited = new Promise((_, reject) => { + stop = () => reject(signal.reason) + signal.addEventListener("abort", stop, { once: true }) + const seconds = dbtAdapterAccessInternals.adapterStartTimeoutMs / 1000 + timer = setTimeout( + () => reject(new Error(`the dbt adapter didn't start within ${seconds} s`)), + dbtAdapterAccessInternals.adapterStartTimeoutMs, + ) + }) + const adapter = await Promise.race([getDbtAdapter(), waited]).finally(() => { + signal.removeEventListener("abort", stop) + clearTimeout(timer) + }) + if (!adapter) { + // No adapter also when the project exists but the adapter failed to start: say which. + const root = await getDbtProjectRoot() + const found = !!root && fs.existsSync(path.join(root, "dbt_project.yml")) + return found + ? { refusal: "not_started", reason: "the dbt adapter didn't start" } + : { refusal: "no_project", reason: "no dbt project here" } + } + if (adapter.isInitialized?.() === false) return { refusal: "not_loaded", reason: "the dbt adapter didn't load the project" } + return { adapter } + } catch (e) { + if (signal.aborted) throw e + return { refusal: "not_started", reason: e instanceof Error ? e.message : String(e) } + } +} + +/** + * The adapter's project root and, when the session's directory is in another dbt project, that one + * (`other`, both links resolved). Undefined when the session's directory is in no dbt project: the + * adapter's own discovery (a dbt.json, or a walk up from the working directory) decides then. + */ +async function sessionProject(): Promise<{ root?: string; other?: string } | undefined> { + const { findProjectRoot } = await import("../../../../dbt-tools/src/config") + const found = findProjectRoot(projectRoot()) + if (!found) return undefined + const { getDbtProjectRoot } = await import("../native/connections/register") + const root = await getDbtProjectRoot() + const real = (p: string) => { + try { + return fs.realpathSync(p) + } catch { + return path.resolve(p) + } + } + const here = real(found) + // path.relative ignores case on win32. + if (!root) return { other: here } + return path.relative(real(root), here) ? { root: real(root), other: here } : { root } +} diff --git a/packages/opencode/src/altimate/tools/dbt-cost-estimate.ts b/packages/opencode/src/altimate/tools/dbt-cost-estimate.ts new file mode 100644 index 0000000000..6529d828c6 --- /dev/null +++ b/packages/opencode/src/altimate/tools/dbt-cost-estimate.ts @@ -0,0 +1,17 @@ +import z from "zod" +import { Tool } from "../../tool/tool" + +export const DbtCostEstimateTool = Tool.define("dbt_cost_estimate", { + description: "Estimate the bytes a dbt model or a SQL query would scan, from a warehouse dry run (BigQuery).", + parameters: z.object({ + model: z.string().optional().describe("Model name to compile and estimate"), + sql: z.string().optional().describe("SQL (may use Jinja) to compile and estimate"), + }), + async execute() { + return { + title: "dbt_cost_estimate: not implemented", + metadata: { success: false }, + output: "dbt_cost_estimate is not implemented in this build of altimate-code yet.", + } + }, +}) diff --git a/packages/opencode/src/altimate/tools/dbt-cte-profile.ts b/packages/opencode/src/altimate/tools/dbt-cte-profile.ts new file mode 100644 index 0000000000..e7a5bde647 --- /dev/null +++ b/packages/opencode/src/altimate/tools/dbt-cte-profile.ts @@ -0,0 +1,17 @@ +import z from "zod" +import { Tool } from "../../tool/tool" + +export const DbtCteProfileTool = Tool.define("dbt_cte_profile", { + description: "List a dbt model's CTEs, or time one CTE with a row-count query.", + parameters: z.object({ + model: z.string().describe("Model name"), + cte: z.string().optional().describe("CTE to time; without it, the CTE list is returned"), + }), + async execute() { + return { + title: "dbt_cte_profile: not implemented", + metadata: { success: false }, + output: "dbt_cte_profile is not implemented in this build of altimate-code yet.", + } + }, +}) diff --git a/packages/opencode/src/altimate/tools/dbt-defer-state.ts b/packages/opencode/src/altimate/tools/dbt-defer-state.ts new file mode 100644 index 0000000000..75cbfe19cf --- /dev/null +++ b/packages/opencode/src/altimate/tools/dbt-defer-state.ts @@ -0,0 +1,16 @@ +import z from "zod" +import { Tool } from "../../tool/tool" + +export const DbtDeferStateTool = Tool.define("dbt_defer_state", { + description: "dbt Power User's defer-to-production setting for a dbt project, and the profile's default target.", + parameters: z.object({ + project_dir: z.string().describe("dbt project folder"), + }), + async execute() { + return { + title: "dbt_defer_state: not implemented", + metadata: { success: false }, + output: "dbt_defer_state is not implemented in this build of altimate-code yet.", + } + }, +}) diff --git a/packages/opencode/src/altimate/tools/dbt-docs-coverage.ts b/packages/opencode/src/altimate/tools/dbt-docs-coverage.ts new file mode 100644 index 0000000000..c4e0ec42d3 --- /dev/null +++ b/packages/opencode/src/altimate/tools/dbt-docs-coverage.ts @@ -0,0 +1,17 @@ +import z from "zod" +import { Tool } from "../../tool/tool" + +export const DbtDocsCoverageTool = Tool.define("dbt_docs_coverage", { + description: "Documentation coverage of a dbt model: which output columns have a description, and whether the model has one.", + parameters: z.object({ + model: z.string().describe("Model name"), + manifest_path: z.string().optional().describe("manifest.json to read (default: the dbt project's target/manifest.json)"), + }), + async execute() { + return { + title: "dbt_docs_coverage: not implemented", + metadata: { success: false }, + output: "dbt_docs_coverage is not implemented in this build of altimate-code yet.", + } + }, +}) diff --git a/packages/opencode/src/altimate/tools/dbt-health.ts b/packages/opencode/src/altimate/tools/dbt-health.ts new file mode 100644 index 0000000000..a3f26352c4 --- /dev/null +++ b/packages/opencode/src/altimate/tools/dbt-health.ts @@ -0,0 +1,16 @@ +import z from "zod" +import { Tool } from "../../tool/tool" + +export const DbtHealthTool = Tool.define("dbt_health", { + description: "DataPilot project health insights for the dbt project.", + parameters: z.object({ + config_path: z.string().optional().describe("DataPilot config file to use"), + }), + async execute() { + return { + title: "dbt_health: not implemented", + metadata: { success: false }, + output: "dbt_health is not implemented in this build of altimate-code yet.", + } + }, +}) diff --git a/packages/opencode/src/altimate/tools/dbt-map-relations.ts b/packages/opencode/src/altimate/tools/dbt-map-relations.ts new file mode 100644 index 0000000000..1216d6b2ee --- /dev/null +++ b/packages/opencode/src/altimate/tools/dbt-map-relations.ts @@ -0,0 +1,18 @@ +import z from "zod" +import { Tool } from "../../tool/tool" + +export const DbtMapRelationsTool = Tool.define("dbt_map_relations", { + description: "Map each table a raw SQL query reads to the dbt ref() or source() for it, from the manifest.", + parameters: z.object({ + sql: z.string().describe("Raw SQL to map"), + dialect: z.string().optional().describe("SQL dialect of the query"), + manifest_path: z.string().optional().describe("manifest.json to match against (default: the dbt project's target/manifest.json)"), + }), + async execute() { + return { + title: "dbt_map_relations: not implemented", + metadata: { success: false }, + output: "dbt_map_relations is not implemented in this build of altimate-code yet.", + } + }, +}) diff --git a/packages/opencode/src/altimate/tools/dbt-run-results.ts b/packages/opencode/src/altimate/tools/dbt-run-results.ts new file mode 100644 index 0000000000..86e68a1668 --- /dev/null +++ b/packages/opencode/src/altimate/tools/dbt-run-results.ts @@ -0,0 +1,16 @@ +import z from "zod" +import { Tool } from "../../tool/tool" + +export const DbtRunResultsTool = Tool.define("dbt_run_results", { + description: "The latest dbt run's results from target/run_results.json: command, duration and each node's status.", + parameters: z.object({ + project_dir: z.string().optional().describe("dbt project folder (default: the dbt project found from the working directory)"), + }), + async execute() { + return { + title: "dbt_run_results: not implemented", + metadata: { success: false }, + output: "dbt_run_results is not implemented in this build of altimate-code yet.", + } + }, +}) diff --git a/packages/opencode/src/altimate/tools/dbt-sql-graph.ts b/packages/opencode/src/altimate/tools/dbt-sql-graph.ts new file mode 100644 index 0000000000..b2af1b0f18 --- /dev/null +++ b/packages/opencode/src/altimate/tools/dbt-sql-graph.ts @@ -0,0 +1,16 @@ +import z from "zod" +import { Tool } from "../../tool/tool" + +export const DbtSqlGraphTool = Tool.define("dbt_sql_graph", { + description: "A dbt model's SQL as a graph: its CTEs, joins and filters, and the edges between them.", + parameters: z.object({ + model: z.string().describe("Model name"), + }), + async execute() { + return { + title: "dbt_sql_graph: not implemented", + metadata: { success: false }, + output: "dbt_sql_graph is not implemented in this build of altimate-code yet.", + } + }, +}) diff --git a/packages/opencode/src/altimate/tools/docs-share-comments.ts b/packages/opencode/src/altimate/tools/docs-share-comments.ts new file mode 100644 index 0000000000..2c8c0a2ab3 --- /dev/null +++ b/packages/opencode/src/altimate/tools/docs-share-comments.ts @@ -0,0 +1,20 @@ +import z from "zod" +import { Tool } from "../../tool/tool" + +export const DocsShareCommentsTool = Tool.define("docs_share_comments", { + description: "Comment threads on shared dbt docs: list open threads, reply to one, or resolve one.", + parameters: z.object({ + action: z.enum(["list", "reply", "resolve"]).describe("What to do"), + project: z.string().optional().describe("dbt project name the shared docs belong to"), + share_id: z.number().int().optional().describe("Shared docs id"), + group_id: z.number().int().optional().describe("Comment thread id"), + text: z.string().optional().describe("Reply text"), + }), + async execute() { + return { + title: "docs_share_comments: not implemented", + metadata: { success: false }, + output: "docs_share_comments is not implemented in this build of altimate-code yet.", + } + }, +}) diff --git a/packages/opencode/test/altimate/dbt-adapter-access.test.ts b/packages/opencode/test/altimate/dbt-adapter-access.test.ts new file mode 100644 index 0000000000..b565e5a58a --- /dev/null +++ b/packages/opencode/test/altimate/dbt-adapter-access.test.ts @@ -0,0 +1,221 @@ +// The tools the dbt Power User features call: their allow-list entries and placeholder answers, the +// one gate every dbt-adapter-backed tool goes through, and the API helper for new endpoint files. The +// dbt adapter is replaced through `dbtAdapterInternals`, so these tests need no Python or dbt. +import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, test } from "bun:test" +import { mkdtemp, rm, writeFile } from "node:fs/promises" +import { realpathSync } from "node:fs" +import os from "node:os" +import path from "node:path" +import { Effect } from "effect" +import type { Tool } from "@/tool/tool" +import { SessionID, MessageID } from "@/session/schema" +import { dbtAdapterInternals, resetDbtAdapter } from "@/altimate/native/connections/register" +import { dbtAdapterAccessInternals, dbtAdapterFor } from "@/altimate/tools/dbt-adapter-access" +import { EXPOSED, MCP_OPEN_WRITE_TOOLS, WRITE_TOOLS } from "@/altimate/tool-cli/allow-list" +import { AltimateApi } from "@/altimate/api/client" +import { Instance } from "@/project/instance" +import { disposeInstance } from "@/effect/instance-registry" +import { initTool } from "./tool-fixture" + +/** Each tool with the smallest arguments its parameters accept. */ +const MINIMAL_ARGS: Record> = { + dbt_run_results: {}, + dbt_defer_state: { project_dir: "." }, + dbt_map_relations: { sql: "select 1" }, + docs_share_comments: { action: "list" }, + dbt_cost_estimate: {}, + dbt_health: {}, + dbt_sql_graph: { model: "customers" }, + dbt_docs_coverage: { model: "customers" }, + dbt_cte_profile: { model: "customers" }, +} +const FEATURE_TOOLS = Object.keys(MINIMAL_ARGS) + +const ctx: Tool.Context = { + sessionID: SessionID.make("ses_dbt_adapter_access"), + messageID: MessageID.make("msg_dbt_adapter_access"), + agent: "analyst", + abort: new AbortController().signal, + messages: [], + metadata: () => Effect.void, + ask: () => Effect.void, +} + +let project: string + +function useConfig() { + dbtAdapterInternals.readConfig = async () => ({ + projectRoot: project, + pythonPath: "python3", + dbtIntegration: "corecommand", + queryLimit: 500, + }) +} + +beforeAll(async () => { + project = await mkdtemp(path.join(os.tmpdir(), "dbt-adapter-access-")) + await writeFile(path.join(project, "dbt_project.yml"), "name: jaffle_shop\n") +}) + +afterAll(async () => { + await rm(project, { recursive: true, force: true }) +}) + +beforeEach(() => resetDbtAdapter()) + +afterEach(() => { + resetDbtAdapter() + delete dbtAdapterInternals.readConfig + delete dbtAdapterInternals.create +}) + +describe("feature tools in the allow-list", () => { + test("all nine are exposed, in the map's alphabetical order, and none is a write tool", () => { + const ids = Object.keys(EXPOSED) + for (const id of FEATURE_TOOLS) { + expect(ids).toContain(id) + expect(WRITE_TOOLS.has(id)).toBe(false) + expect(MCP_OPEN_WRITE_TOOLS.has(id)).toBe(false) + } + const listed = ids.filter((id) => id.startsWith("dbt_") || id.startsWith("docs_")) + expect(listed).toEqual([...listed].sort()) + }) + + test("each placeholder answers that it isn't implemented, as a failure", async () => { + for (const id of FEATURE_TOOLS) { + const tool = await initTool(EXPOSED[id] as any) + expect(tool.id).toBe(id) + const result = await tool.execute(MINIMAL_ARGS[id], ctx) + expect(result.title).toBe(`${id}: not implemented`) + expect(result.metadata.success).toBe(false) + expect(result.output).toContain(id) + } + }) +}) + +describe("dbtAdapterFor", () => { + let created: number + beforeEach(() => { + created = 0 + useConfig() + dbtAdapterInternals.create = async () => { + created += 1 + return { isInitialized: () => true, dispose: async () => {} } + } + }) + + test("refuses a read-only call and never creates the adapter", async () => { + const access = await dbtAdapterFor({ ...ctx, extra: { forceReadOnly: true } }) + expect(access.adapter).toBeUndefined() + expect(access.refusal).toBe("read_only") + expect(access.reason).toContain("--allow-write") + expect(created).toBe(0) + }) + + test("refuses a plan-mode call", async () => { + const access = await dbtAdapterFor({ ...ctx, agent: "plan" }) + expect(access.refusal).toBe("read_only") + expect(created).toBe(0) + }) + + test("returns the adapter where writes are allowed", async () => { + const access = await dbtAdapterFor(ctx) + expect(access.refusal).toBeUndefined() + expect(access.adapter?.isInitialized()).toBe(true) + expect(created).toBe(1) + }) + + test("says when the adapter didn't load the project", async () => { + dbtAdapterInternals.create = async () => ({ isInitialized: () => false, dispose: async () => {} }) + const access = await dbtAdapterFor(ctx) + expect(access).toEqual({ refusal: "not_loaded", reason: "the dbt adapter didn't load the project" }) + }) + + test("says there is no dbt project when the config finds none", async () => { + dbtAdapterInternals.readConfig = async () => null + const access = await dbtAdapterFor(ctx) + expect(access).toEqual({ refusal: "no_project", reason: "no dbt project here" }) + }) + + test("says when the adapter doesn't start in time", async () => { + dbtAdapterInternals.create = () => new Promise(() => {}) + const saved = dbtAdapterAccessInternals.adapterStartTimeoutMs + dbtAdapterAccessInternals.adapterStartTimeoutMs = 20 + try { + const access = await dbtAdapterFor(ctx) + expect(access).toEqual({ refusal: "not_started", reason: "the dbt adapter didn't start within 0.02 s" }) + } finally { + dbtAdapterAccessInternals.adapterStartTimeoutMs = saved + } + }) + + test("a cancelled call throws rather than answering", async () => { + const controller = new AbortController() + controller.abort() + await expect(dbtAdapterFor({ ...ctx, abort: controller.signal })).rejects.toThrow() + expect(created).toBe(0) + }) + + test("a session in a dbt project the adapter isn't set up for says so", async () => { + dbtAdapterInternals.readConfig = async () => null + try { + const access = await Instance.provide({ directory: project, fn: () => dbtAdapterFor(ctx) }) + expect(access).toEqual({ + refusal: "other_project", + reason: `the dbt adapter isn't set up for ${realpathSync(project)}`, + }) + expect(created).toBe(0) + } finally { + await disposeInstance(project) + } + }) +}) + +describe("AltimateApi.apiRequest", () => { + let home: string + let saved: string | undefined + beforeEach(async () => { + home = await mkdtemp(path.join(os.tmpdir(), "altimate-api-home-")) + saved = process.env.OPENCODE_TEST_HOME + process.env.OPENCODE_TEST_HOME = home + }) + afterEach(async () => { + if (saved === undefined) delete process.env.OPENCODE_TEST_HOME + else process.env.OPENCODE_TEST_HOME = saved + await rm(home, { recursive: true, force: true }) + }) + + test("fails with a clear message when no credentials are stored", async () => { + await expect(AltimateApi.apiRequest("GET", "/dbt/v1/x")).rejects.toThrow( + `No Altimate credentials are stored: sign in to Altimate in altimate-code first (they are saved to ${path.join(home, ".altimate", "altimate.json")}).`, + ) + }) + + test("sends the stored key and tenant and returns the JSON body", async () => { + await Bun.write( + path.join(home, ".altimate", "altimate.json"), + JSON.stringify({ altimateUrl: "https://api.example.test/", altimateInstanceName: "acme", altimateApiKey: "k1" }), + ) + const original = globalThis.fetch + const seen: { url: string; init: RequestInit }[] = [] + globalThis.fetch = (async (url: string, init: RequestInit) => { + seen.push({ url, init }) + return new Response(JSON.stringify({ ok: 1 }), { status: 200 }) + }) as unknown as typeof fetch + try { + expect(await AltimateApi.apiRequest("POST", "/dbt/v1/x", { a: 1 })).toEqual({ ok: 1 }) + expect(seen[0].url).toBe("https://api.example.test/dbt/v1/x") + const headers = seen[0].init.headers as Record + expect(headers.Authorization).toBe("Bearer k1") + expect(headers["x-tenant"]).toBe("acme") + expect(seen[0].init.body).toBe(JSON.stringify({ a: 1 })) + } finally { + globalThis.fetch = original + } + }) + + test("names the file when the stored credentials can't be read", async () => { + await Bun.write(path.join(home, ".altimate", "altimate.json"), "{not json") + await expect(AltimateApi.apiRequest("GET", "/dbt/v1/x")).rejects.toThrow("can't be used") + }) +}) From 6f445a6f55336fda69433e4973ab47d01b8f065e Mon Sep 17 00:00:00 2001 From: Harish Kumar Date: Fri, 13 Nov 2026 16:53:38 +0530 Subject: [PATCH 02/26] feat: dbt_cte_profile lists a model's CTEs and times one CTE with a count query Ports the CTE finder and count-query builder: it reads the model's raw text, skips comments and string literals, and handles quoted names and column lists. Without `cte` the tool lists the CTEs and runs nothing; with `cte` it runs SELECT COUNT(*) over that CTE and the earlier ones of its WITH clause through the dbt adapter and returns the milliseconds and row count. Co-Authored-By: Claude Sonnet 5.5 --- .../opencode/src/altimate/tools/cte-detect.ts | 398 +++++ .../src/altimate/tools/dbt-cte-profile.ts | 168 +- .../opencode/test/altimate/cte-detect.test.ts | 1492 +++++++++++++++++ .../test/altimate/dbt-adapter-access.test.ts | 2 + .../test/altimate/dbt-cte-profile.test.ts | 202 +++ 5 files changed, 2255 insertions(+), 7 deletions(-) create mode 100644 packages/opencode/src/altimate/tools/cte-detect.ts create mode 100644 packages/opencode/test/altimate/cte-detect.test.ts create mode 100644 packages/opencode/test/altimate/dbt-cte-profile.test.ts diff --git a/packages/opencode/src/altimate/tools/cte-detect.ts b/packages/opencode/src/altimate/tools/cte-detect.ts new file mode 100644 index 0000000000..33b64b6a08 --- /dev/null +++ b/packages/opencode/src/altimate/tools/cte-detect.ts @@ -0,0 +1,398 @@ +/** + * CTE detection and count-query building for the CTE profiler. + * + * Works on the raw model text (Jinja included) with string offsets. The scanner + * skips SQL line/block comments, Jinja comments and string literals, and + * understands quoted CTE names and CTE column lists. + */ + +export interface CteInfo { + /** Identifier as written, including a trailing column list such as `my_cte (id, name)`. */ + name: string + /** Offset where the name starts. */ + nameStart: number + /** Offset just after the name (and its column list). */ + nameEnd: number + /** Offset of the first character after the CTE body's opening parenthesis. */ + queryStart: number + /** Offset of the CTE body's closing parenthesis. */ + queryEnd: number + /** Order of the CTE within its WITH clause. */ + index: number + /** Offset of the WITH keyword that opens this CTE's clause. */ + withClauseStart: number +} + +export interface CteProfileEntry { + name: string + line: number + queryTimeMs: number + marginalTimeMs: number + rowCount: number + tier: "hot" | "warm" | "cool" +} + +/** Longest quoted identifier ("name", `name`, [name]) the identifier regex accepts. */ +const MAX_QUOTED_IDENTIFIER_LENGTH = 500 + +/** Longest CTE column list the identifier regex accepts. */ +const MAX_COLUMN_LIST_LENGTH = 1000 + +const IDENT_PART = + `(?:[a-zA-Z_][a-zA-Z0-9_]*|"[^"]{1,${MAX_QUOTED_IDENTIFIER_LENGTH}}"|` + + `\`[^\`]{1,${MAX_QUOTED_IDENTIFIER_LENGTH}}\`|` + + `\\[[^\\]]{1,${MAX_QUOTED_IDENTIFIER_LENGTH}}\\])` + +function identifierRegex(): RegExp { + return new RegExp(`(${IDENT_PART}(?:\\.${IDENT_PART})*(?:\\s*\\([^)]{0,${MAX_COLUMN_LIST_LENGTH}}\\))?)`, "gi") +} + +interface StringState { + newPos: number + inString: boolean + stringChar: string +} + +function handleSqlStringLiteral(text: string, pos: number, inString: boolean, stringChar: string): StringState { + const char = text[pos] + const nextChar = pos < text.length - 1 ? text[pos + 1] : "" + + if (!inString && (char === "'" || char === '"')) { + return { newPos: pos, inString: true, stringChar: char } + } + if (inString && char === stringChar) { + if (nextChar === stringChar) { + // Doubled quote is an escaped quote: skip the second one. + return { newPos: pos + 1, inString: true, stringChar } + } + return { newPos: pos, inString: false, stringChar: "" } + } + return { newPos: pos, inString, stringChar } +} + +/** + * When `pos` starts a comment (`--`, block, or Jinja), return the position of + * its last character (the newline for line comments); otherwise return `pos`. + */ +function handleSqlComment(text: string, pos: number): number { + const char = text[pos] + const nextChar = pos < text.length - 1 ? text[pos + 1] : "" + + if (char === "-" && nextChar === "-") { + let endPos = pos + 2 + while (endPos < text.length && text[endPos] !== "\n" && text[endPos] !== "\r") endPos++ + return endPos + } + + if (char === "/" && nextChar === "*") { + let endPos = pos + 2 + while (endPos < text.length - 1) { + if (text[endPos] === "*" && text[endPos + 1] === "/") return endPos + 1 + endPos++ + } + return text.length + } + + if (char === "{" && nextChar === "#") { + let endPos = pos + 2 + while (endPos < text.length - 1) { + if (text[endPos] === "#" && text[endPos + 1] === "}") return endPos + 1 + endPos++ + } + return text.length + } + + return pos +} + +function isPositionInsideComment(content: string, position: number): boolean { + let pos = 0 + let inBlockComment = false + let inJinjaComment = false + + while (pos < position && pos < content.length) { + const char = content[pos] + const nextChar = pos < content.length - 1 ? content[pos + 1] : "" + + if (!inBlockComment && !inJinjaComment && char === "/" && nextChar === "*") { + inBlockComment = true + pos += 2 + continue + } + if (inBlockComment && char === "*" && nextChar === "/") { + inBlockComment = false + pos += 2 + continue + } + if (!inBlockComment && !inJinjaComment && char === "{" && nextChar === "#") { + inJinjaComment = true + pos += 2 + continue + } + if (inJinjaComment && char === "#" && nextChar === "}") { + inJinjaComment = false + pos += 2 + continue + } + if (!inBlockComment && !inJinjaComment && char === "-" && nextChar === "-") { + const lineStart = pos + let lineEnd = pos + 2 + while (lineEnd < content.length && content[lineEnd] !== "\n" && content[lineEnd] !== "\r") lineEnd++ + if (position >= lineStart && position < lineEnd) return true + pos = lineEnd + continue + } + pos++ + } + + return inBlockComment || inJinjaComment +} + +/** Every WITH keyword outside comments and strings. */ +function findWithKeywords(text: string): number[] { + const positions: number[] = [] + let pos = 0 + let inString = false + let stringChar = "" + + while (pos < text.length) { + const commentEnd = handleSqlComment(text, pos) + if (commentEnd !== pos) { + pos = commentEnd + 1 + continue + } + + const s = handleSqlStringLiteral(text, pos, inString, stringChar) + pos = s.newPos + inString = s.inString + stringChar = s.stringChar + + if (!inString) { + const m = text.substring(pos).match(/^with\b/i) + if (m) { + const before = pos > 0 ? text[pos - 1] : " " + if (!/[a-zA-Z0-9_]/.test(before)) positions.push(pos) + pos += m[0].length + continue + } + } + pos++ + } + return positions +} + +function findMatchingClosingParen(text: string, openParenPos: number): number { + let parenCount = 1 + let pos = openParenPos + 1 + let inString = false + let stringChar = "" + + while (pos < text.length && parenCount > 0) { + const char = text[pos] + + const commentEnd = handleSqlComment(text, pos) + if (commentEnd !== pos) { + pos = commentEnd + continue + } + + const s = handleSqlStringLiteral(text, pos, inString, stringChar) + pos = s.newPos + inString = s.inString + stringChar = s.stringChar + + if (!inString) { + if (char === "(") parenCount++ + else if (char === ")") parenCount-- + } + pos++ + } + + return parenCount === 0 ? pos - 1 : -1 +} + +/** Position of the main SELECT after the CTEs, the end of text, or -1 when a nested WITH is found. */ +function findWithClauseEnd(text: string, withStartPos: number): number { + let pos = withStartPos + let parenCount = 0 + let inString = false + let stringChar = "" + + while (pos < text.length) { + const char = text[pos] + + const commentEnd = handleSqlComment(text, pos) + if (commentEnd !== pos) { + pos = commentEnd + continue + } + + const s = handleSqlStringLiteral(text, pos, inString, stringChar) + pos = s.newPos + inString = s.inString + stringChar = s.stringChar + + if (!inString) { + if (char === "(") { + parenCount++ + } else if (char === ")") { + parenCount-- + } else if (parenCount === 0) { + const rest = text.substring(pos) + if (/^\s*with\b/i.test(rest)) return -1 + if (/^\s*select\b/i.test(rest)) return pos + } + } + pos++ + } + return text.length +} + +/** `identifier [(columns)] [comments] AS (` starting at `startIndex`; returns the matched text or undefined. */ +function validateCteMatchWithComments(content: string, startIndex: number, identifierName: string): string | undefined { + let pos = startIndex + identifierName.length + + while (pos < content.length && /\s/.test(content[pos])) pos++ + if (pos < content.length && content[pos] === "(") { + const end = findMatchingClosingParen(content, pos) + if (end !== -1) pos = end + 1 + } + + while (pos < content.length) { + while (pos < content.length && /\s/.test(content[pos])) pos++ + if (pos >= content.length) break + + const commentEnd = handleSqlComment(content, pos) + if (commentEnd !== pos) { + pos = commentEnd + 1 + continue + } + + const asMatch = content.substring(pos).match(/^as\s*\(/i) + if (asMatch) return content.substring(startIndex, pos + asMatch[0].length) + break + } + return undefined +} + +function findCteIdentifiersOnly(withClauseContent: string): Array<{ index: number; name: string; fullMatch: string }> { + const regex = identifierRegex() + const out: Array<{ index: number; name: string; fullMatch: string }> = [] + let match: RegExpExecArray | null + + while ((match = regex.exec(withClauseContent)) !== null) { + const name = match[1] + if (isPositionInsideComment(withClauseContent, match.index)) continue + const fullMatch = validateCteMatchWithComments(withClauseContent, match.index, name) + if (fullMatch !== undefined) out.push({ index: match.index, name, fullMatch }) + } + return out +} + +function extractCtesFromWithClause( + withClauseContent: string, + withStartPos: number, + withClauseStart: number, + ctes: CteInfo[], +): void { + let cteIndex = 0 + + for (const m of findCteIdentifiersOnly(withClauseContent)) { + const nameStart = withStartPos + m.index + const nameEnd = nameStart + m.name.length + const openParenRel = m.index + m.fullMatch.length - 1 + const closeRel = findMatchingClosingParen(withClauseContent, openParenRel) + if (closeRel === -1) continue + + ctes.push({ + name: m.name, + nameStart, + nameEnd, + queryStart: withStartPos + openParenRel + 1, + queryEnd: withStartPos + closeRel, + index: cteIndex, + withClauseStart, + }) + cteIndex++ + } +} + +/** Find every CTE in the model text, in source order. */ +export function detectCtes(text: string): CteInfo[] { + const ctes: CteInfo[] = [] + + for (const withPos of findWithKeywords(text)) { + let start = withPos + 4 // "with".length + + while (start < text.length) { + while (start < text.length && /\s/.test(text[start])) start++ + if (start >= text.length) break + const commentEnd = handleSqlComment(text, start) + if (commentEnd !== start) { + start = commentEnd + 1 + continue + } + break + } + + const end = findWithClauseEnd(text, start) + if (end === -1) continue + + extractCtesFromWithClause(text.substring(start, end), start, withPos, ctes) + } + + return ctes +} + +/** Drop a trailing column list: `my_cte (id, name)` -> `my_cte`. */ +export function stripColumnList(rawName: string): string { + return rawName.replace(/\s*\([^)]*\)\s*$/, "").trim() +} + +export function quoteSqlIdentifier(identifier: string): string { + if (identifier.match(/^["'`[]/) || identifier.includes(".")) return identifier + if (!identifier.match(/^[a-zA-Z_][a-zA-Z0-9_]*$/)) return `"${identifier}"` + return identifier +} + +/** + * `SELECT COUNT(*)` over one CTE, keeping the preamble (dbt configs, variables) + * and every earlier CTE of the same WITH clause, so the query compiles on its own. + */ +export function buildCountQuery(text: string, ctes: CteInfo[], target: CteInfo): string | undefined { + const upToTarget = ctes.filter((c) => c.withClauseStart === target.withClauseStart && c.index <= target.index) + + const definitions: string[] = [] + for (const cte of upToTarget) { + if (cte.queryStart <= cte.nameStart) continue + // Slice from the name to the body so column lists and comments before AS are kept. + const header = text.substring(cte.nameStart, cte.queryStart) + const body = text.substring(cte.queryStart, cte.queryEnd) + definitions.push(`${header}\n${body}\n)`) + } + if (definitions.length === 0) return undefined + + const preamble = text.substring(0, target.withClauseStart).trim() + let query = "" + if (preamble) query += preamble + "\n\n" + query += "WITH " + definitions.join(",\n") + query += `\nSELECT COUNT(*) AS _profile_count FROM ${quoteSqlIdentifier(stripColumnList(target.name))}` + return query +} + +/** + * Read the single COUNT(*) value by position: Snowflake and Oracle upper-case + * the alias, so a lookup by name would report 0 rows. + */ +export function extractRowCount(data: Record[] | undefined): number { + if (!data || data.length === 0) return 0 + const first = Object.values(data[0])[0] + return typeof first === "number" ? first : Number(first) || 0 +} + +/** Zero-based line number of an offset. */ +export function lineOf(text: string, offset: number): number { + let line = 0 + for (let i = 0; i < offset && i < text.length; i++) if (text[i] === "\n") line++ + return line +} diff --git a/packages/opencode/src/altimate/tools/dbt-cte-profile.ts b/packages/opencode/src/altimate/tools/dbt-cte-profile.ts index e7a5bde647..8d085c0ed5 100644 --- a/packages/opencode/src/altimate/tools/dbt-cte-profile.ts +++ b/packages/opencode/src/altimate/tools/dbt-cte-profile.ts @@ -1,17 +1,171 @@ +import fs from "fs" +import path from "path" import z from "zod" import { Tool } from "../../tool/tool" +import { loadRawManifest, findModel } from "../native/dbt/helpers" +import { projectRoot } from "../native/connections/registry" +import { dbtAdapterFor } from "./dbt-adapter-access" +import { buildCountQuery, detectCtes, extractRowCount, lineOf, stripColumnList, type CteInfo } from "./cte-detect" + +/** Folders a model search never enters. */ +const SKIPPED_DIRS = new Set(["target", "dbt_packages", "dbt_modules", "node_modules", "logs", "venv", "site-packages"]) + +/** Deepest folder level the model search follows. */ +const MAX_SEARCH_DEPTH = 12 + +/** Largest model file the tool reads. */ +const MAX_MODEL_BYTES = 4 * 1024 * 1024 + +/** A CTE name without its column list, quotes or letter case, for comparing what the caller typed. */ +function nameKey(name: string): string { + return stripColumnList(name) + .replace(/^["'`[]|["'`\]]$/g, "") + .toLowerCase() +} + +function inside(root: string, file: string): boolean { + const rel = path.relative(root, file) + return rel !== "" && !rel.startsWith("..") && !path.isAbsolute(rel) +} + +function realpath(p: string): string { + try { + return fs.realpathSync(p) + } catch { + return path.resolve(p) + } +} + +/** Every `.sql` under the project, outside the build folders. */ +function findSqlFiles(root: string, name: string): string[] { + const found: string[] = [] + const walk = (dir: string, depth: number) => { + let entries: fs.Dirent[] + try { + entries = fs.readdirSync(dir, { withFileTypes: true }) + } catch { + return + } + for (const entry of entries) { + if (entry.name.startsWith(".")) continue + const full = path.join(dir, entry.name) + if (entry.isDirectory()) { + if (depth < MAX_SEARCH_DEPTH && !SKIPPED_DIRS.has(entry.name)) walk(full, depth + 1) + } else if (entry.isFile() && entry.name === `${name}.sql`) { + found.push(full) + } + } + } + walk(root, 0) + return found.sort() +} + +/** The model's source file: a project-relative path, the manifest's record of it, or a search by name. */ +function modelFile(root: string, model: string): { file?: string; error?: string } { + const realRoot = realpath(root) + if (/\.sql$/i.test(model) || /[\\/]/.test(model)) { + const candidate = path.resolve(realRoot, model) + if (!inside(realRoot, realpath(candidate))) return { error: `${model} is outside the dbt project` } + return fs.existsSync(candidate) ? { file: candidate } : { error: `no file ${model} in the dbt project` } + } + try { + const manifest = loadRawManifest(path.join(realRoot, "target", "manifest.json")) + const node = manifest?.nodes ? findModel(manifest.nodes, model) : null + if (node?.original_file_path) { + const candidate = path.resolve(realRoot, node.original_file_path) + if (inside(realRoot, realpath(candidate)) && fs.existsSync(candidate)) return { file: candidate } + } + } catch { + // A manifest that can't be read doesn't stop the search by name. + } + const name = model.startsWith("model.") ? (model.split(".").pop() ?? model) : model + const matches = findSqlFiles(realRoot, name) + if (matches.length === 1) return { file: matches[0] } + if (matches.length === 0) return { error: `no model named ${model} in the dbt project` } + const rel = matches.map((m) => path.relative(realRoot, m)) + return { error: `${model} matches ${matches.length} files (${rel.join(", ")}): pass the path` } +} + +async function projectDir(): Promise { + const { getDbtProjectRoot } = await import("../native/connections/register") + const root = await getDbtProjectRoot() + if (root) return root + const here = projectRoot() + return fs.existsSync(path.join(here, "dbt_project.yml")) ? here : undefined +} + +function failure(title: string, output: string, extra: Record = {}) { + return { title: `dbt_cte_profile: ${title}`, metadata: { success: false, ...extra }, output } +} export const DbtCteProfileTool = Tool.define("dbt_cte_profile", { - description: "List a dbt model's CTEs, or time one CTE with a row-count query.", + description: + "List a dbt model's CTEs, or time one CTE with a row-count query. Without `cte` it returns the CTE names in source order and runs nothing (a read-only call works). With `cte` it runs `SELECT COUNT(*)` over that CTE and the CTEs before it in the same WITH clause on the warehouse, and returns the elapsed milliseconds and the row count; that run needs write permission (`--allow-write`) because it starts the dbt adapter and compiles the model. Each run recomputes the earlier CTEs, so a model of N CTEs costs N queries.", parameters: z.object({ - model: z.string().describe("Model name"), + model: z + .string() + .describe("Model name, unique_id, or project-relative path to the model's .sql file (e.g. 'customers' or 'models/marts/customers.sql')"), cte: z.string().optional().describe("CTE to time; without it, the CTE list is returned"), }), - async execute() { - return { - title: "dbt_cte_profile: not implemented", - metadata: { success: false }, - output: "dbt_cte_profile is not implemented in this build of altimate-code yet.", + async execute(args, ctx) { + const root = await projectDir() + if (!root) return failure("no dbt project", "No dbt project here: run from a dbt project folder.") + const located = modelFile(root, args.model) + if (!located.file) return failure("model not found", located.error ?? `no model named ${args.model}`) + const size = fs.statSync(located.file).size + if (size > MAX_MODEL_BYTES) return failure("model too large", `${args.model} is ${size} bytes; the largest file read is ${MAX_MODEL_BYTES}.`) + const text = fs.readFileSync(located.file, "utf-8") + const ctes = detectCtes(text) + const file = path.relative(realpath(root), located.file) + + if (!args.cte) { + const list = ctes.map((c) => ({ + name: stripColumnList(c.name), + line: lineOf(text, c.nameStart) + 1, + index: c.index, + clause: c.withClauseStart, + })) + const output = list.length + ? [`${list.length} CTE${list.length === 1 ? "" : "s"} in ${file}:`, ...list.map((c, i) => `${i + 1}. ${c.name} (line ${c.line})`)].join("\n") + : `No CTEs in ${file}.` + return { + title: `dbt_cte_profile: ${list.length} CTE${list.length === 1 ? "" : "s"} in ${args.model}`, + metadata: { success: true, model: args.model, file, ctes: list }, + output, + } + } + + const wanted = nameKey(args.cte) + const matches = ctes.filter((c) => nameKey(c.name) === wanted) + if (matches.length === 0) { + const names = ctes.map((c) => stripColumnList(c.name)).join(", ") + return failure("CTE not found", `${args.model} has no CTE named ${args.cte}${names ? `; it has: ${names}` : ""}.`, { model: args.model }) + } + const target: CteInfo = matches[0] + const query = buildCountQuery(text, ctes, target) + if (!query) return failure("no query", `Could not build a count query for ${args.cte}.`, { model: args.model, cte: args.cte }) + + const access = await dbtAdapterFor(ctx) + if (!access.adapter) { + return failure("not run", `${access.reason}`, { model: args.model, cte: args.cte, refusal: access.refusal }) + } + ctx.abort.throwIfAborted() + const name = stripColumnList(target.name) + const started = performance.now() + try { + const result = await access.adapter.immediatelyExecuteSQLWithLimit(query, `cte_profiler_${name}`, 1) + const queryMs = Math.round(performance.now() - started) + const rows = extractRowCount(result?.data) + const note = matches.length > 1 ? ` (${matches.length} CTEs share this name; the first was timed)` : "" + return { + title: `dbt_cte_profile: ${name} ${queryMs} ms`, + metadata: { success: true, model: args.model, cte: name, index: target.index, queryMs, rows }, + output: `${name}: ${queryMs} ms, ${rows} row${rows === 1 ? "" : "s"}${note}`, + } + } catch (e) { + if (ctx.abort.aborted) throw e + const message = e instanceof Error ? e.message : String(e) + return failure("query failed", `The count query for ${name} failed: ${message}`, { model: args.model, cte: name }) } }, }) diff --git a/packages/opencode/test/altimate/cte-detect.test.ts b/packages/opencode/test/altimate/cte-detect.test.ts new file mode 100644 index 0000000000..8f7b86c372 --- /dev/null +++ b/packages/opencode/test/altimate/cte-detect.test.ts @@ -0,0 +1,1492 @@ +import { describe, expect, it } from "bun:test" +import { + buildCountQuery, + detectCtes as detectCtesInText, + extractRowCount, + lineOf, + quoteSqlIdentifier, + stripColumnList, + type CteInfo, +} from "../../src/altimate/tools/cte-detect" + +// Detector cases ported from the dbt Power User CTE detector suite. A "document" is the model text. +describe("detectCtes", () => { + const detectCtes = (document: string): CteInfo[] => detectCtesInText(document) + + const assertCTE = (cte: CteInfo, expectedName: string, expectedIndex: number, _description?: string) => { + expect(cte.name).toBe(expectedName) + expect(cte.index).toBe(expectedIndex) + expect(typeof cte.nameStart).toBe("number") + expect(typeof cte.queryStart).toBe("number") + expect(typeof cte.withClauseStart).toBe("number") + } + + describe("Basic CTE Detection", () => { + it("should detect single CTE correctly with proper ranges", () => { + const sql = "with my_cte as (select 1 as id) select * from my_cte" + const document = sql + + const ctes = detectCtes(document) + + expect(ctes).toHaveLength(1) + assertCTE(ctes[0], "my_cte", 0, "CTE with simple select and ID column") + }) + + it("should detect multiple CTEs correctly with proper indexing", () => { + const sql = `with + first_cte as (select 1 as id), + second_cte as (select 2 as id), + third_cte as (select 3 as id) + select * from third_cte` + const document = sql + + const ctes = detectCtes(document) + + expect(ctes).toHaveLength(3) + assertCTE(ctes[0], "first_cte", 0, "CTE with simple select") + assertCTE(ctes[1], "second_cte", 1, "CTE with different ID value") + assertCTE(ctes[2], "third_cte", 2, "CTE with third ID value") + }) + + it("should handle CTEs with column lists", () => { + const sql = "with my_cte (id, name) as (select 1, 'test') select * from my_cte" + const document = sql + + const ctes = detectCtes(document) + + expect(ctes).toHaveLength(1) + assertCTE(ctes[0], "my_cte (id, name)", 0, "CTE with explicit column list specification") + }) + + it("should handle quoted CTE names", () => { + const sql = `with "my cte" as (select 1 as id) select * from "my cte"` + const document = sql + + const ctes = detectCtes(document) + + expect(ctes).toHaveLength(1) + assertCTE(ctes[0], '"my cte"', 0, "CTE with double-quoted name containing space") + }) + }) + + describe("Comments in CTE Body", () => { + it("should handle line comments with unmatched quotes in CTE body", () => { + const sql = `with + cte_without_lens as ( + select * + from {{ ref('parent') }} + where + condition = true -- CTE doesn't have a lens + ), + cte_with_lens as ( + select * + from {{ ref('parent') }} + where + condition = true -- CTE has a 'lens' + ) + select * from cte_with_lens` + + const document = sql + const ctes = detectCtes(document) + + expect(ctes).toHaveLength(2) + assertCTE(ctes[0], "cte_without_lens", 0, "CTE with line comment containing unmatched single quote") + assertCTE(ctes[1], "cte_with_lens", 1, "CTE with line comment containing matched single quotes") + }) + + it("should handle block comments with unmatched quotes in CTE body", () => { + const sql = `with + cte_with_block_comment as ( + select * + from {{ ref('parent') }} + where + condition = true /* This is a block comment with unmatched quote ' */ + ) + select * from cte_with_block_comment` + + const document = sql + const ctes = detectCtes(document) + + expect(ctes).toHaveLength(1) + assertCTE(ctes[0], "cte_with_block_comment", 0, "CTE with block comment containing unmatched single quote") + }) + + it("should handle Jinja comments with unmatched quotes in CTE body", () => { + const sql = `with + cte_with_jinja_comment as ( + select * + from {{ ref('parent') }} + where + condition = true {# This is a Jinja comment with unmatched quote ' #} + ) + select * from cte_with_jinja_comment` + + const document = sql + const ctes = detectCtes(document) + + expect(ctes).toHaveLength(1) + assertCTE(ctes[0], "cte_with_jinja_comment", 0, "CTE with Jinja comment containing unmatched single quote") + }) + }) + + describe("Comments Between CTE Name and AS", () => { + it("should handle single-line block comment between name and AS", () => { + const sql = "with source /* test */ as (select 1 as id) select * from source" + const document = sql + + const ctes = detectCtes(document) + + expect(ctes).toHaveLength(1) + assertCTE(ctes[0], "source", 0, "CTE with single-line block comment between name and AS keyword") + // Skip content extraction test for now due to Range/Position mocking complexity + // The important part is that CTE detection works correctly + }) + + it("should handle multi-line block comment between name and AS", () => { + const sql = `with source +/* this is +a multi-line comment +that should be ignored */ +as ( + select * from {{ ref('raw_orders') }} +) +select * from source` + + const document = sql + const ctes = detectCtes(document) + + expect(ctes).toHaveLength(1) + assertCTE(ctes[0], "source", 0, "CTE with multi-line block comment between name and AS keyword") + // Skip content extraction test for now due to Range/Position mocking complexity + // The important part is that CTE detection works correctly + }) + + it("should handle Jinja comment between name and AS", () => { + const sql = "with target {# some comment #} as (select 1 as id) select * from target" + const document = sql + + const ctes = detectCtes(document) + + expect(ctes).toHaveLength(1) + assertCTE(ctes[0], "target", 0, "CTE with Jinja comment between name and AS keyword") + }) + + it("should handle line comment between name and AS (multiline)", () => { + const sql = `with multiline_cte -- this is a comment + as ( + select 4 as id + ) + select * from multiline_cte` + + const document = sql + const ctes = detectCtes(document) + + expect(ctes).toHaveLength(1) + assertCTE(ctes[0], "multiline_cte", 0, "CTE with line comment on separate line before AS keyword") + }) + + it("should handle complex comment with quotes between name and AS", () => { + const sql = `with complex_comment /* more complex comment with 'quotes' */ as ( + select 5 as id + ) select * from complex_comment` + + const document = sql + const ctes = detectCtes(document) + + expect(ctes).toHaveLength(1) + assertCTE(ctes[0], "complex_comment", 0, "CTE with block comment containing both single and double quotes") + }) + }) + + describe("Comments Before and Around CTE Names", () => { + it("should handle block comment before CTE name", () => { + const sql = "with /* comment before name */ my_cte as (select 1 as id) select * from my_cte" + const document = sql + + const ctes = detectCtes(document) + + expect(ctes).toHaveLength(1) + assertCTE(ctes[0], "my_cte", 0, "CTE with block comment before name") + }) + + it("should handle line comment before CTE name on separate line", () => { + const sql = `with + -- This is a comment before the CTE name + my_cte as ( + select 1 as id + ) + select * from my_cte` + const document = sql + + const ctes = detectCtes(document) + + expect(ctes).toHaveLength(1) + assertCTE(ctes[0], "my_cte", 0, "CTE with line comment before name on separate line") + }) + + it("should handle Jinja comment before CTE name", () => { + const sql = "with {# comment before name #} my_cte as (select 1 as id) select * from my_cte" + const document = sql + + const ctes = detectCtes(document) + + expect(ctes).toHaveLength(1) + assertCTE(ctes[0], "my_cte", 0, "CTE with Jinja comment before name") + }) + + it("should handle comment immediately after WITH keyword", () => { + const sql = "with /* comment after WITH */ my_cte as (select 1 as id) select * from my_cte" + const document = sql + + const ctes = detectCtes(document) + + expect(ctes).toHaveLength(1) + assertCTE(ctes[0], "my_cte", 0, "CTE with block comment immediately after WITH keyword") + }) + + it("should handle line comment after WITH keyword", () => { + const sql = `with -- comment after WITH + my_cte as ( + select 1 as id + ) + select * from my_cte` + const document = sql + + const ctes = detectCtes(document) + + expect(ctes).toHaveLength(1) + assertCTE(ctes[0], "my_cte", 0, "CTE with line comment after WITH keyword") + }) + + it("should handle multiple comments around CTE name", () => { + const sql = "with /* before */ my_cte /* after */ as (select 1 as id) select * from my_cte" + const document = sql + + const ctes = detectCtes(document) + + expect(ctes).toHaveLength(1) + assertCTE(ctes[0], "my_cte", 0, "CTE with block comments before and after name") + }) + + it("should handle mixed comment types around CTE name", () => { + const sql = `with -- line comment + /* block comment */ my_cte {# jinja comment #} as ( + select 1 as id + ) + select * from my_cte` + const document = sql + + const ctes = detectCtes(document) + + expect(ctes).toHaveLength(1) + assertCTE(ctes[0], "my_cte", 0, "CTE with mixed comment types around name") + }) + + it("should handle comments in multi-CTE scenarios", () => { + const sql = `with + /* comment */ first_cte as ( + select 1 as id + ), + -- another comment + second_cte /* inline comment */ as ( + select 2 as id + ), + {# jinja comment #} third_cte as ( + select 3 as id + ) + select * from third_cte` + const document = sql + + const ctes = detectCtes(document) + + expect(ctes).toHaveLength(3) + assertCTE(ctes[0], "first_cte", 0, "First CTE with block comment before name") + assertCTE(ctes[1], "second_cte", 1, "Second CTE with line and inline comments") + assertCTE(ctes[2], "third_cte", 2, "Third CTE with Jinja comment before name") + }) + + it("should handle comments with special characters around CTE name", () => { + const sql = "with /* comment with $pecial ch@rs & symbols! */ my_cte as (select 1 as id) select * from my_cte" + const document = sql + + const ctes = detectCtes(document) + + expect(ctes).toHaveLength(1) + assertCTE(ctes[0], "my_cte", 0, "CTE with comment containing special characters") + }) + + it("should handle unterminated block comment before CTE name", () => { + const sql = "with /* unterminated comment my_cte as (select 1 as id) select * from my_cte" + const document = sql + + const ctes = detectCtes(document) + + // With unterminated comment, the parser may not detect the CTE correctly + // This test verifies graceful handling of malformed comments + expect(ctes.length).toBeGreaterThanOrEqual(0) + }) + + it("should handle sequential line and block comments between name and AS", () => { + const sql = `with source -- test +/* this is +a multi-line comment +that should be ignored */ +as ( + select * from {{ ref('raw_orders') }} +) +select * from renamed` + + const document = sql + const ctes = detectCtes(document) + + expect(ctes).toHaveLength(1) + assertCTE(ctes[0], "source", 0, "CTE with sequential line and block comments between name and AS") + }) + }) + + describe("Complex Comment Scenarios", () => { + it("should handle CTE with mixed block and Jinja comments", () => { + const sql = `with +source /* test */ as( + {#- + Normally we would select from the table here, but we are using seeds to load + our data in this project + #} + select * from {{ ref('raw_orders') }} +), + +renamed as ( + select + id as order_id, + user_id as customer_id, + order_date, + status + from source +) + +select * from renamed` + + const document = sql + const ctes = detectCtes(document) + + expect(ctes).toHaveLength(2) + assertCTE(ctes[0], "source", 0, "CTE with block comment after name and Jinja comment in body") + assertCTE(ctes[1], "renamed", 1, "CTE that references another CTE with column aliasing") + }) + + it("should handle CTE with multi-line block comment before AS keyword", () => { + const sql = `with source +/* this is +a multi-line comment +that should be ignored */ +as ( + select * from {{ ref('raw_orders') }} +), + +renamed as ( + select + id as order_id, + user_id as customer_id, + order_date, + status + from source +) + +select * from renamed` + + const document = sql + const ctes = detectCtes(document) + + expect(ctes).toHaveLength(2) + assertCTE(ctes[0], "source", 0, "CTE with multi-line block comment spanning multiple lines before AS") + assertCTE(ctes[1], "renamed", 1, "CTE that selects from another CTE with column renaming") + }) + }) + + describe("Very Lengthy Comments with Quoted Names", () => { + it("should handle extremely long block comment before quoted CTE name", () => { + // Create a 1500+ character comment to test real-world lengthy comment scenarios + const longComment = + "/* " + + "This is an extremely long comment that might appear in real-world SQL files. ".repeat(25) + + "It contains multiple sentences and potentially problematic content like 'quoted strings', " + + "special characters !@#$%^&*()_+-=[]{}|;:,.<>?, and even problematic keywords " + + "that might confuse the parser. This comment also contains backslashes \\ and forward slashes // " + + "and other regex-problematic characters like .*+?^${}()|[] that could cause catastrophic backtracking " + + "in poorly designed regular expressions. The comment goes on and on longer. ".repeat(10) + + " */" + + const sql = `with ${longComment} "quoted_cte_name" as (select 1 as id) select * from "quoted_cte_name"` + const document = sql + + const ctes = detectCtes(document) + + expect(ctes).toHaveLength(1) + assertCTE( + ctes[0], + '"quoted_cte_name"', + 0, + "CTE with extremely long block comment (1500+ chars) before quoted name", + ) + }) + + it("should handle extremely long line comment before quoted CTE name", () => { + // Create a 1000+ character line comment + const longLineComment = + "-- " + + "This is an extremely long line comment that might appear in real-world SQL files. ".repeat(12) + + "It contains 'quoted strings' and special characters that could cause issues. " + + const sql = `with\n${longLineComment}\n"quoted_cte_name" as (select 1 as id) select * from "quoted_cte_name"` + const document = sql + + const ctes = detectCtes(document) + + expect(ctes).toHaveLength(1) + assertCTE( + ctes[0], + '"quoted_cte_name"', + 0, + "CTE with extremely long line comment (1000+ chars) before quoted name", + ) + }) + + it("should handle extremely long Jinja comment before quoted CTE name", () => { + // Create a 1200+ character Jinja comment + const longJinjaComment = + "{# " + + "This is an extremely long Jinja comment that might appear in real-world dbt files. ".repeat(15) + + "It contains 'quoted strings', template variables like {{ var('my_var') }}, and complex logic. " + + "The comment includes special characters and database keywords that could confuse parsers. " + + " #}" + + const sql = `with ${longJinjaComment} "quoted_cte_name" as (select 1 as id) select * from "quoted_cte_name"` + const document = sql + + const ctes = detectCtes(document) + + expect(ctes).toHaveLength(1) + assertCTE( + ctes[0], + '"quoted_cte_name"', + 0, + "CTE with extremely long Jinja comment (1200+ chars) before quoted name", + ) + }) + + it("should handle extremely long block comment after quoted CTE name", () => { + // Create a 2000+ character comment after the name + const longComment = + "/* " + + "This is an extremely long comment that appears after the table name but before AS. ".repeat(30) + + "It contains multiple paragraphs, 'quoted strings', and complex descriptions that might " + + "explain the business logic behind the transformation. The comment includes database keywords, special " + + "characters, and other potentially problematic content for regex parsing. ".repeat(5) + + " */" + + const sql = `with "quoted_cte_name" ${longComment} as (select 1 as id) select * from "quoted_cte_name"` + const document = sql + + const ctes = detectCtes(document) + + expect(ctes).toHaveLength(1) + assertCTE( + ctes[0], + '"quoted_cte_name"', + 0, + "CTE with extremely long block comment (2000+ chars) after quoted name", + ) + }) + + it("should handle multiple very long comments around quoted CTE name", () => { + // Create multiple long comments around the CTE name + const longCommentBefore = + "/* " + "Long comment before the CTE name. ".repeat(20) + "Contains 'quotes' and special characters. " + " */" + + const longCommentAfter = + "/* " + + "Long comment after the CTE name but before AS. ".repeat(20) + + "Also contains 'quotes' and special characters. " + + " */" + + const sql = `with ${longCommentBefore} "quoted_cte_name" ${longCommentAfter} as (select 1 as id) select * from "quoted_cte_name"` + const document = sql + + const ctes = detectCtes(document) + + expect(ctes).toHaveLength(1) + assertCTE(ctes[0], '"quoted_cte_name"', 0, "CTE with multiple very long comments around quoted name") + }) + + it("should handle extremely long mixed comment types around quoted CTE name", () => { + // Test with line comment, block comment, and Jinja comment (avoid SQL keywords in comments) + const longLineComment = "-- " + "Very long line comment containing 'quotes' and special characters. ".repeat(10) + + const longBlockComment = + "/* " + "Very long block comment containing 'quotes' and special characters. ".repeat(15) + " */" + + const longJinjaComment = + "{# " + "Very long Jinja comment containing 'quotes' and template logic. ".repeat(10) + " #}" + + const sql = `${longLineComment}\n${longBlockComment}\nwith "quoted_cte_name" ${longJinjaComment} as (select 1 as id) select * from "quoted_cte_name"` + const document = sql + + const ctes = detectCtes(document) + + expect(ctes).toHaveLength(1) + assertCTE(ctes[0], '"quoted_cte_name"', 0, "CTE with extremely long mixed comment types around quoted name") + }) + + it("should handle lengthy comments with quoted names in multi-CTE scenario", () => { + // Test performance with multiple CTEs each having lengthy comments + const longComment1 = + "/* " + + "First data source containing very long description. ".repeat(25) + + "Contains 'quotes' and special characters for testing. " + + " */" + + const longComment2 = + "/* " + + "Second data source containing very long description. ".repeat(25) + + "Also contains 'quotes' and special characters for testing. " + + " */" + + const sql = `with + ${longComment1} "first_quoted_cte" as ( + select 1 as id, 'first' as name + ), + ${longComment2} "second_quoted_cte" as ( + select 2 as id, 'second' as name + ) + select * from "first_quoted_cte" union all select * from "second_quoted_cte"` + + const document = sql + + const ctes = detectCtes(document) + + expect(ctes).toHaveLength(2) + assertCTE(ctes[0], '"first_quoted_cte"', 0, "First CTE with very long comment before quoted name") + assertCTE(ctes[1], '"second_quoted_cte"', 1, "Second CTE with very long comment before quoted name") + }) + + it("should handle extremely long comments with regex-problematic content", () => { + // Create a comment with content that could cause regex issues + const problemComment = + "/* " + + "This comment contains regex-problematic content: .*+?^${}()|[] ".repeat(20) + + "It also has nested quotes like 'text with \"nested quotes\" inside' and " + + "backslashes \\ and forward slashes // that could break poorly designed regex. " + + "The comment includes SQL injection attempts like '; DROP TABLE users; -- " + + "and other malicious content that should be handled safely. ".repeat(10) + + " */" + + const sql = `with ${problemComment} "secure_cte_name" as (select 1 as id) select * from "secure_cte_name"` + const document = sql + + const ctes = detectCtes(document) + + expect(ctes).toHaveLength(1) + assertCTE(ctes[0], '"secure_cte_name"', 0, "CTE with extremely long comment containing regex-problematic content") + }) + }) + + describe("MAX_QUOTED_IDENTIFIER_LENGTH Edge Cases", () => { + it("should handle comments with quoted strings longer than MAX_QUOTED_IDENTIFIER_LENGTH", () => { + // Create a quoted string in comment that's longer than 500 chars + const veryLongQuotedString = '"' + "a".repeat(600) + '"' + const commentWithLongQuotedString = `/* This comment contains a very long quoted string: ${veryLongQuotedString} that exceeds the MAX_QUOTED_IDENTIFIER_LENGTH limit */` + + const sql = `with ${commentWithLongQuotedString} "real_cte" as (select 1 as id) select * from "real_cte"` + const document = sql + + const ctes = detectCtes(document) + + expect(ctes).toHaveLength(1) + assertCTE( + ctes[0], + '"real_cte"', + 0, + "CTE with comment containing quoted string longer than MAX_QUOTED_IDENTIFIER_LENGTH", + ) + }) + + it("should handle quoted CTE name approaching MAX_QUOTED_IDENTIFIER_LENGTH limit", () => { + // Create a quoted CTE name that's close to but under the 500 char limit + const longCTEName = '"' + "very_long_cte_name_" + "x".repeat(450) + '"' + + const sql = `with ${longCTEName} as (select 1 as id) select * from ${longCTEName}` + const document = sql + + const ctes = detectCtes(document) + + expect(ctes).toHaveLength(1) + assertCTE(ctes[0], longCTEName, 0, "CTE with quoted name approaching MAX_QUOTED_IDENTIFIER_LENGTH limit") + }) + + it("should handle quoted CTE name exactly at MAX_QUOTED_IDENTIFIER_LENGTH limit", () => { + // Create a quoted CTE name that's exactly at the 500 char limit (498 chars inside quotes + 2 quotes = 500) + const exactLimitCTEName = '"' + "x".repeat(498) + '"' + + const sql = `with ${exactLimitCTEName} as (select 1 as id) select * from ${exactLimitCTEName}` + const document = sql + + const ctes = detectCtes(document) + + expect(ctes).toHaveLength(1) + assertCTE(ctes[0], exactLimitCTEName, 0, "CTE with quoted name exactly at MAX_QUOTED_IDENTIFIER_LENGTH limit") + }) + + it("should handle comment with multiple long quoted strings and real CTE", () => { + // Test multiple long quoted strings in comments + const longQuotedString1 = '"' + "fake_identifier_1_" + "a".repeat(600) + '"' + const longQuotedString2 = '"' + "fake_identifier_2_" + "b".repeat(700) + '"' + const commentWithMultipleLongQuotes = `/* Comment with ${longQuotedString1} and also ${longQuotedString2} */` + + const sql = `with ${commentWithMultipleLongQuotes} "actual_cte" as (select 1 as id) select * from "actual_cte"` + const document = sql + + const ctes = detectCtes(document) + + expect(ctes).toHaveLength(1) + assertCTE( + ctes[0], + '"actual_cte"', + 0, + "CTE with comment containing multiple quoted strings longer than MAX_QUOTED_IDENTIFIER_LENGTH", + ) + }) + + it("should handle extremely long comment with quoted content and real long CTE name", () => { + // Combine very long comment with quoted content AND a long CTE name + const longQuotedInComment = '"' + "fake_cte_name_in_comment_" + "z".repeat(800) + '"' + const longComment = `/* ${"Very long comment text. ".repeat(50)} Contains fake quoted identifier: ${longQuotedInComment} ${"More comment text. ".repeat(30)} */` + const longRealCTEName = '"' + "real_cte_name_" + "r".repeat(400) + '"' + + const sql = `with ${longComment} ${longRealCTEName} as (select 1 as id) select * from ${longRealCTEName}` + const document = sql + + const ctes = detectCtes(document) + + expect(ctes).toHaveLength(1) + assertCTE( + ctes[0], + longRealCTEName, + 0, + "CTE with extremely long comment containing long quoted string and long real CTE name", + ) + }) + + it("should handle line comment with long quoted string before quoted CTE", () => { + // Test line comment with long quoted content + const longQuotedInLineComment = '"' + "fake_line_comment_identifier_" + "l".repeat(600) + '"' + const lineComment = `-- Line comment with long quoted string: ${longQuotedInLineComment}` + + const sql = `with\n${lineComment}\n"real_cte" as (select 1 as id) select * from "real_cte"` + const document = sql + + const ctes = detectCtes(document) + + expect(ctes).toHaveLength(1) + assertCTE( + ctes[0], + '"real_cte"', + 0, + "CTE with line comment containing quoted string longer than MAX_QUOTED_IDENTIFIER_LENGTH", + ) + }) + + it("should handle Jinja comment with long quoted string before quoted CTE", () => { + // Test Jinja comment with long quoted content + const longQuotedInJinjaComment = '"' + "fake_jinja_identifier_" + "j".repeat(650) + '"' + const jinjaComment = `{# Jinja comment with long quoted string: ${longQuotedInJinjaComment} #}` + + const sql = `with ${jinjaComment} "real_cte" as (select 1 as id) select * from "real_cte"` + const document = sql + + const ctes = detectCtes(document) + + expect(ctes).toHaveLength(1) + assertCTE( + ctes[0], + '"real_cte"', + 0, + "CTE with Jinja comment containing quoted string longer than MAX_QUOTED_IDENTIFIER_LENGTH", + ) + }) + + it("should handle mixed comment types with various length quoted strings", () => { + // Test combination of different comment types with various length quoted strings + const shortQuoted = '"short"' + const mediumQuoted = '"' + "medium_" + "m".repeat(200) + '"' + const longQuoted = '"' + "very_long_" + "v".repeat(800) + '"' + + const lineComment = `-- Line comment with ${shortQuoted}` + const blockComment = `/* Block comment with ${mediumQuoted} and ${longQuoted} */` + const jinjaComment = `{# Jinja comment with ${longQuoted} #}` + + const sql = `${lineComment}\n${blockComment}\nwith ${jinjaComment} "mixed_test_cte" as (select 1 as id) select * from "mixed_test_cte"` + const document = sql + + const ctes = detectCtes(document) + + expect(ctes).toHaveLength(1) + assertCTE(ctes[0], '"mixed_test_cte"', 0, "CTE with mixed comment types containing various length quoted strings") + }) + }) + + describe("CTE Column List Test Cases", () => { + describe("Basic Column List Variations", () => { + it("should handle simple column list with multiple columns", () => { + const sql = + "with my_cte (id, name, email, created_at) as (select 1, 'John', 'john@example.com', now()) select * from my_cte" + const document = sql + + const ctes = detectCtes(document) + + expect(ctes).toHaveLength(1) + assertCTE(ctes[0], "my_cte (id, name, email, created_at)", 0, "CTE with multiple column names in list") + }) + + it("should handle column list with quoted column names", () => { + const sql = + 'with my_cte ("user id", "full name", "email address") as (select 1, \'John Doe\', \'john@example.com\') select * from my_cte' + const document = sql + + const ctes = detectCtes(document) + + expect(ctes).toHaveLength(1) + assertCTE( + ctes[0], + 'my_cte ("user id", "full name", "email address")', + 0, + "CTE with quoted column names containing spaces", + ) + }) + + it("should handle column list with mixed quoted and unquoted names", () => { + const sql = + "with my_cte (id, \"user name\", email, \"created at\") as (select 1, 'John', 'john@example.com', now()) select * from my_cte" + const document = sql + + const ctes = detectCtes(document) + + expect(ctes).toHaveLength(1) + assertCTE( + ctes[0], + 'my_cte (id, "user name", email, "created at")', + 0, + "CTE with mixed quoted and unquoted column names", + ) + }) + + it("should handle column list with simple data types", () => { + const sql = + "with my_cte (id integer, name varchar, amount decimal) as (select 1, 'John', 100.50) select * from my_cte" + const document = sql + + const ctes = detectCtes(document) + + expect(ctes).toHaveLength(1) + assertCTE( + ctes[0], + "my_cte (id integer, name varchar, amount decimal)", + 0, + "CTE with simple column names and data types", + ) + }) + + it("should handle column list with single column", () => { + const sql = "with my_cte (single_column) as (select 'value') select * from my_cte" + const document = sql + + const ctes = detectCtes(document) + + expect(ctes).toHaveLength(1) + assertCTE(ctes[0], "my_cte (single_column)", 0, "CTE with single column in list") + }) + }) + + describe("Length and Performance Edge Cases", () => { + it("should handle column list approaching MAX_COLUMN_LIST_LENGTH limit", () => { + // Create a column list that's close to but under the 1000 char limit + const longColumnList = Array.from({ length: 20 }, (_, i) => `column_${i}_with_long_name`).join(", ") + const sql = `with my_cte (${longColumnList}) as (select ${Array.from({ length: 20 }, (_, i) => i).join(", ")}) select * from my_cte` + const document = sql + + const ctes = detectCtes(document) + + expect(ctes).toHaveLength(1) + assertCTE( + ctes[0], + `my_cte (${longColumnList})`, + 0, + "CTE with column list approaching MAX_COLUMN_LIST_LENGTH limit", + ) + }) + + it("should handle column list near MAX_COLUMN_LIST_LENGTH limit", () => { + // Create a column list that's close to the 1000 char limit + const nearLimitColumnList = "x".repeat(900) // 900 chars - well within limit + const sql = `with my_cte (${nearLimitColumnList}) as (select 1) select * from my_cte` + const document = sql + + const ctes = detectCtes(document) + + expect(ctes).toHaveLength(1) + assertCTE( + ctes[0], + `my_cte (${nearLimitColumnList})`, + 0, + "CTE with column list near MAX_COLUMN_LIST_LENGTH limit", + ) + }) + + it("should handle very long individual column names", () => { + const longColumnName1 = '"' + "very_long_column_name_" + "a".repeat(200) + '"' + const longColumnName2 = '"' + "another_long_column_name_" + "b".repeat(200) + '"' + const sql = `with my_cte (${longColumnName1}, ${longColumnName2}) as (select 1, 2) select * from my_cte` + const document = sql + + const ctes = detectCtes(document) + + expect(ctes).toHaveLength(1) + assertCTE( + ctes[0], + `my_cte (${longColumnName1}, ${longColumnName2})`, + 0, + "CTE with very long individual column names", + ) + }) + }) + + describe("Complex Column List Scenarios", () => { + it("should handle column list with nested parentheses (function calls)", () => { + const sql = + "with my_cte (id, calculated_value, formatted_date) as (select 1, round(avg(amount), 2), to_char(created_at, 'YYYY-MM-DD')) select * from my_cte" + const document = sql + + const ctes = detectCtes(document) + + expect(ctes).toHaveLength(1) + assertCTE( + ctes[0], + "my_cte (id, calculated_value, formatted_date)", + 0, + "CTE with column list containing nested parentheses from function calls", + ) + }) + + it("should handle column list with simple constraints", () => { + const sql = + "with my_cte (id not null, name, amount default 0) as (select 1, 'John', 100.0) select * from my_cte" + const document = sql + + const ctes = detectCtes(document) + + expect(ctes).toHaveLength(1) + assertCTE( + ctes[0], + "my_cte (id not null, name, amount default 0)", + 0, + "CTE with column list containing simple constraints", + ) + }) + + it("should handle column list with line breaks and formatting", () => { + const sql = `with my_cte ( + id, + name, + email, + created_at + ) as ( + select 1, 'John', 'john@example.com', now() + ) select * from my_cte` + const document = sql + + const ctes = detectCtes(document) + + expect(ctes).toHaveLength(1) + assertCTE( + ctes[0], + `my_cte ( + id, + name, + email, + created_at + )`, + 0, + "CTE with multi-line formatted column list", + ) + }) + + it("should handle column list with special characters and escaped quotes", () => { + const sql = `with my_cte ("col_with_'quote", "col with spaces", "col@with#symbols") as (select 1, 2, 3) select * from my_cte` + const document = sql + + const ctes = detectCtes(document) + + expect(ctes).toHaveLength(1) + assertCTE( + ctes[0], + `my_cte ("col_with_'quote", "col with spaces", "col@with#symbols")`, + 0, + "CTE with column list containing special characters and quotes", + ) + }) + }) + + describe("Column List + Comment Combinations", () => { + it("should handle comments before the opening parenthesis", () => { + const sql = "with my_cte /* comment before column list */ (id, name) as (select 1, 'John') select * from my_cte" + const document = sql + + const ctes = detectCtes(document) + + // Current limitation: Comments between CTE name and column list break the parsing + // This is an edge case that would require more complex parsing logic + expect(ctes).toHaveLength(0) + }) + + it("should handle comments after the closing parenthesis", () => { + const sql = "with my_cte (id, name) /* comment after column list */ as (select 1, 'John') select * from my_cte" + const document = sql + + const ctes = detectCtes(document) + + expect(ctes).toHaveLength(1) + assertCTE(ctes[0], "my_cte (id, name)", 0, "CTE with comment after column list closing parenthesis") + }) + + it("should handle comments between column list and AS keyword", () => { + const sql = `with my_cte (id, name, email) + -- This is a comment before AS + /* Another comment */ + as (select 1, 'John', 'john@example.com') select * from my_cte` + const document = sql + + const ctes = detectCtes(document) + + expect(ctes).toHaveLength(1) + assertCTE(ctes[0], "my_cte (id, name, email)", 0, "CTE with comments between column list and AS keyword") + }) + + it("should handle mixed comment types around column lists", () => { + const sql = `with /* before */ my_cte ( + id, -- line comment + name /* inline comment */, + email + ) {# jinja comment #} as (select 1, 'John', 'john@example.com') select * from my_cte` + const document = sql + + const ctes = detectCtes(document) + + expect(ctes).toHaveLength(1) + // Comments within column lists are currently included in the CTE name + // This is expected behavior given the current parsing approach + assertCTE( + ctes[0], + `my_cte ( + id, -- line comment + name /* inline comment */, + email + )`, + 0, + "CTE with mixed comment types around column list", + ) + }) + }) + + describe("Error Handling and Edge Cases", () => { + it("should handle empty column lists", () => { + const sql = "with my_cte () as (select 1) select * from my_cte" + const document = sql + + const ctes = detectCtes(document) + + expect(ctes).toHaveLength(1) + assertCTE(ctes[0], "my_cte ()", 0, "CTE with empty column list") + }) + + it("should handle column lists with only whitespace", () => { + const sql = "with my_cte ( ) as (select 1) select * from my_cte" + const document = sql + + const ctes = detectCtes(document) + + expect(ctes).toHaveLength(1) + assertCTE(ctes[0], "my_cte ( )", 0, "CTE with whitespace-only column list") + }) + + it("should handle column lists in combination with quoted CTE names", () => { + const sql = + 'with "my cte" (id, "user name", email) as (select 1, \'John\', \'john@example.com\') select * from "my cte"' + const document = sql + + const ctes = detectCtes(document) + + expect(ctes).toHaveLength(1) + assertCTE( + ctes[0], + '"my cte" (id, "user name", email)', + 0, + "CTE with quoted name and column list with quoted columns", + ) + }) + + it("should handle deeply nested parentheses in column definitions", () => { + const sql = + "with my_cte (calculated_field, nested_calc) as (select sum(case when (x > 0 and (y < 10 or z = (a + b))) then 1 else 0 end), avg(coalesce(field1, field2, 0))) select * from my_cte" + const document = sql + + const ctes = detectCtes(document) + + expect(ctes).toHaveLength(1) + assertCTE( + ctes[0], + "my_cte (calculated_field, nested_calc)", + 0, + "CTE with column list and deeply nested parentheses in query", + ) + }) + }) + + describe("Real-World Complex Scenarios", () => { + it("should handle simple BigQuery-style column definitions", () => { + const sql = `with my_cte ( + user_id INT64, + user_name STRING, + created_at TIMESTAMP + ) as ( + select 1, 'John', current_timestamp() + ) select * from my_cte` + const document = sql + + const ctes = detectCtes(document) + + expect(ctes).toHaveLength(1) + assertCTE( + ctes[0], + `my_cte ( + user_id INT64, + user_name STRING, + created_at TIMESTAMP + )`, + 0, + "CTE with simple BigQuery-style column definitions", + ) + }) + + it("should handle multi-CTE scenarios with varying column list complexity", () => { + const sql = `with + simple_cte (id, name) as ( + select 1, 'John' + ), + complex_cte ( + user_id, + "full name", + calculated_field, + metadata_json + ) as ( + select id, name, id * 100, '{"key": "value"}' + from simple_cte + ), + no_columns_cte as ( + select * from complex_cte + ) + select * from no_columns_cte` + const document = sql + + const ctes = detectCtes(document) + + expect(ctes).toHaveLength(3) + assertCTE(ctes[0], "simple_cte (id, name)", 0, "First CTE with simple column list") + assertCTE( + ctes[1], + `complex_cte ( + user_id, + "full name", + calculated_field, + metadata_json + )`, + 1, + "Second CTE with complex multi-line column list", + ) + assertCTE(ctes[2], "no_columns_cte", 2, "Third CTE without column list") + }) + }) + }) + + describe("CTE Cross-References", () => { + it("should handle simple sequential CTE references", () => { + const sql = `with + base_data as ( + select 1 as id, 'test' as name + ), + filtered_data as ( + select * from base_data where id > 0 + ), + final_data as ( + select id, upper(name) as name from filtered_data + ) + select * from final_data` + + const document = sql + const ctes = detectCtes(document) + + expect(ctes).toHaveLength(3) + assertCTE(ctes[0], "base_data", 0, "CTE with hardcoded test data") + assertCTE(ctes[1], "filtered_data", 1, "CTE filtering data from base_data CTE") + assertCTE(ctes[2], "final_data", 2, "CTE transforming data from filtered_data CTE") + }) + + it("should handle CTE references with joins", () => { + const sql = `with + orders as ( + select 1 as order_id, 100 as customer_id + ), + customers as ( + select 100 as id, 'John' as name + ), + order_details as ( + select o.order_id, c.name + from orders o + join customers c on o.customer_id = c.id + ) + select * from order_details` + + const document = sql + const ctes = detectCtes(document) + + expect(ctes).toHaveLength(3) + assertCTE(ctes[0], "orders", 0, "CTE with order and customer ID data") + assertCTE(ctes[1], "customers", 1, "CTE with customer ID and name data") + assertCTE(ctes[2], "order_details", 2, "CTE joining orders and customers CTEs") + }) + + it("should handle CTE references with comments and quotes", () => { + const sql = `with + source_data as ( + select id, name -- 'source' data extraction + from raw_table + ), + cleaned_data as ( + /* Reference to 'source_data' CTE */ + select * from source_data + where name is not null + ) + select * from cleaned_data` + + const document = sql + const ctes = detectCtes(document) + + expect(ctes).toHaveLength(2) + assertCTE(ctes[0], "source_data", 0, "CTE with line comment containing single quotes") + assertCTE(ctes[1], "cleaned_data", 1, "CTE referencing source_data with block comment containing quotes") + }) + + it("should handle multiple CTE references in one query", () => { + const sql = `with + cte_a as ( + select 1 as id, 'A' as source + ), + cte_b as ( + select 2 as id, 'B' as source + ), + cte_combined as ( + select * from cte_a + union all + select * from cte_b + ) + select * from cte_combined` + + const document = sql + const ctes = detectCtes(document) + + expect(ctes).toHaveLength(3) + assertCTE(ctes[0], "cte_a", 0, "CTE with ID 1 and source A") + assertCTE(ctes[1], "cte_b", 1, "CTE with ID 2 and source B") + assertCTE(ctes[2], "cte_combined", 2, "CTE combining cte_a and cte_b with UNION ALL") + }) + + it("should handle nested CTE references with subqueries", () => { + const sql = `with + base as ( + select id, amount from transactions + ), + aggregated as ( + select id, sum(amount) as total + from base + group by id + ), + ranked as ( + select *, + (select count(*) from aggregated a2 where a2.total > aggregated.total) as rank + from aggregated + ) + select * from ranked` + + const document = sql + const ctes = detectCtes(document) + + expect(ctes).toHaveLength(3) + assertCTE(ctes[0], "base", 0, "CTE selecting ID and amount from transactions table") + assertCTE(ctes[1], "aggregated", 1, "CTE grouping base CTE data by ID with sum") + assertCTE(ctes[2], "ranked", 2, "CTE adding rank using correlated subquery on aggregated CTE") + }) + }) + + describe("Edge Cases", () => { + it("should handle no CTEs", () => { + const sql = "select * from my_table" + const document = sql + + const ctes = detectCtes(document) + + expect(ctes).toHaveLength(0) + }) + + it("should handle empty WITH clause", () => { + const sql = "with select * from my_table" + const document = sql + + const ctes = detectCtes(document) + + expect(ctes).toHaveLength(0) + }) + + it("should handle complex nested queries", () => { + const sql = `with my_cte as ( + select id, ( + select count(*) + from other_table + where other_table.id = main_table.id + ) as count + from main_table + ) select * from my_cte` + + const document = sql + const ctes = detectCtes(document) + + expect(ctes).toHaveLength(1) + assertCTE(ctes[0], "my_cte", 0, "CTE with correlated subquery counting from other table") + + // Skip content extraction test - focus on CTE detection correctness + }) + + it("should handle unterminated comments gracefully", () => { + const sql = "with source /* unterminated comment as (select 1) select * from source" + const document = sql + + const ctes = detectCtes(document) + + // The regex actually detects "comment" as a CTE name due to the unterminated comment + // This is expected behavior - the regex tries to parse what it can + expect(ctes.length).toBeGreaterThanOrEqual(0) + }) + }) + + describe("CTE Range and Content Validation", () => { + it("should provide correct ranges for CTE name and query", () => { + const sql = "with my_cte as (select 1 as id) select * from my_cte" + const document = sql + + const ctes = detectCtes(document) + + expect(ctes).toHaveLength(1) + const cte = ctes[0] + + // Verify basic range structure is defined + expect(typeof cte.nameStart).toBe("number") + expect(typeof cte.queryStart).toBe("number") + expect(cte.nameStart).toBeDefined() + expect(cte.nameEnd).toBeDefined() + expect(cte.queryStart).toBeDefined() + expect(cte.queryEnd).toBeDefined() + + // Verify the CTE name was extracted correctly + expect(cte.name).toBe("my_cte") + expect(cte.index).toBe(0) + expect(typeof cte.withClauseStart).toBe("number") + }) + + it("should detect complex CTEs correctly", () => { + const sql = `with + orders_summary as ( + select + customer_id, + count(*) as order_count, + sum(total_amount) as total_spent + from orders + where status = 'completed' + group by customer_id + ) + select * from orders_summary` + + const document = sql + const ctes = detectCtes(document) + + expect(ctes).toHaveLength(1) + assertCTE(ctes[0], "orders_summary", 0, "CTE with aggregated customer order data") + + // Verify range structure is correct + expect(typeof ctes[0].queryStart).toBe("number") + }) + + it("should handle CTE detection with comments correctly", () => { + const sql = `with my_cte as ( + select + id, + name, -- This is a comment with quotes 'inside' + /* Another comment */ + status + from users + {# Jinja comment with 'quotes' #} + where active = true + ) select * from my_cte` + + const document = sql + const ctes = detectCtes(document) + + expect(ctes).toHaveLength(1) + assertCTE(ctes[0], "my_cte", 0, "CTE with line, block, and Jinja comments containing quotes") + + // Verify that CTE was detected despite comments with quotes + expect(ctes[0].name).toBe("my_cte") + expect(typeof ctes[0].queryStart).toBe("number") + }) + }) +}) + +describe("detectCtes offsets", () => { + it("returns offsets that slice the name and the body", () => { + const sql = "with my_cte as (select 1 as id) select * from my_cte" + const [cte] = detectCtesInText(sql) + expect(sql.slice(cte.nameStart, cte.nameEnd)).toBe("my_cte") + expect(sql.slice(cte.queryStart, cte.queryEnd)).toBe("select 1 as id") + expect(cte.withClauseStart).toBe(0) + }) + + it("keeps the column list in the name", () => { + const sql = "with my_cte (id, name) as (select 1, 'test') select * from my_cte" + const [cte] = detectCtesInText(sql) + expect(cte.name).toBe("my_cte (id, name)") + expect(stripColumnList(cte.name)).toBe("my_cte") + }) + + it("does not detect a WITH inside a comment or string", () => { + const sql = "-- with fake as (select 1)\nselect 'with x as (select 2)' as s from t" + expect(detectCtesInText(sql)).toEqual([]) + }) + + it("skips a WITH clause that contains another top-level WITH and keeps the inner one", () => { + const sql = "with a as (select 1) with b as (select 2) select * from b" + expect(detectCtesInText(sql).map((c) => c.name)).toEqual(["b"]) + }) + + it("tracks CTE of nested WITH inside a body without confusing the outer index", () => { + const sql = "with outer_cte as (with inner_cte as (select 1 as x) select x from inner_cte) select * from outer_cte" + const names = detectCtesInText(sql).map((c) => c.name) + expect(names).toContain("outer_cte") + }) + + it("reports zero-based lines", () => { + const sql = "with a as (select 1),\n b as (select 2)\nselect * from b" + const ctes = detectCtesInText(sql) + expect(ctes.map((c) => lineOf(sql, c.nameStart))).toEqual([0, 1]) + }) +}) + +describe("buildCountQuery", () => { + const sql = [ + "{{ config(materialized='table') }}", + "", + "with orders as (select * from {{ ref('stg_orders') }}),", + "customers as (select * from {{ ref('stg_customers') }}),", + "joined as (select o.id from orders o join customers c on o.cid = c.id)", + "select * from joined", + ].join("\n") + + it("keeps the preamble and every earlier CTE up to the target", () => { + const ctes = detectCtesInText(sql) + expect(ctes.map((c) => c.name)).toEqual(["orders", "customers", "joined"]) + const query = buildCountQuery(sql, ctes, ctes[1])! + expect(query.startsWith("{{ config(materialized='table') }}")).toBe(true) + expect(query).toContain("WITH orders as (") + expect(query).toContain("customers as (") + expect(query).not.toContain("joined") + expect(query.endsWith("SELECT COUNT(*) AS _profile_count FROM customers")).toBe(true) + }) + + it("counts the last CTE with all of its inputs", () => { + const ctes = detectCtesInText(sql) + const query = buildCountQuery(sql, ctes, ctes[2])! + expect(query).toContain("joined as (") + expect(query).toContain("from orders o join customers c") + expect(query.endsWith("FROM joined")).toBe(true) + }) + + it("keeps a column list in the definition but not in the FROM clause", () => { + const text = "with a (x, y) as (select 1, 2) select * from a" + const ctes = detectCtesInText(text) + const query = buildCountQuery(text, ctes, ctes[0])! + expect(query).toContain("WITH a (x, y) as (") + expect(query.endsWith("FROM a")).toBe(true) + }) + + it("only includes CTEs of the target's own WITH clause", () => { + const text = "with a as (select 1) select * from a;\nwith b as (select 2) select * from b" + const ctes = detectCtesInText(text) + const second = ctes.find((c) => c.name === "b")! + const query = buildCountQuery(text, ctes, second)! + expect(query).not.toContain("WITH a") + expect(query).toContain("WITH b as (") + }) + + it("returns undefined for a CTE that is not in the list", () => { + expect( + buildCountQuery("select 1", [], { + name: "x", + nameStart: 0, + nameEnd: 1, + queryStart: 0, + queryEnd: 0, + index: 0, + withClauseStart: 0, + }), + ).toBeUndefined() + }) +}) + +describe("quoteSqlIdentifier", () => { + it("leaves plain, quoted and dotted names alone and quotes the rest", () => { + expect(quoteSqlIdentifier("my_cte")).toBe("my_cte") + expect(quoteSqlIdentifier('"My Cte"')).toBe('"My Cte"') + expect(quoteSqlIdentifier("`x`")).toBe("`x`") + expect(quoteSqlIdentifier("[x]")).toBe("[x]") + expect(quoteSqlIdentifier("a.b")).toBe("a.b") + expect(quoteSqlIdentifier("my-cte")).toBe('"my-cte"') + }) + + it("is applied to a quoted CTE name in the count query", () => { + const text = 'with "My Cte" as (select 1) select * from "My Cte"' + const ctes = detectCtesInText(text) + expect(buildCountQuery(text, ctes, ctes[0])!.endsWith('FROM "My Cte"')).toBe(true) + }) +}) + +describe("extractRowCount", () => { + it("reads the count by position for upper- and lower-case aliases", () => { + expect(extractRowCount([{ _PROFILE_COUNT: 99 }])).toBe(99) + expect(extractRowCount([{ _profile_count: 42 }])).toBe(42) + }) + + it("coerces string counts", () => { + expect(extractRowCount([{ _PROFILE_COUNT: "1234" }])).toBe(1234) + }) + + it("returns 0 for no rows, a missing value or a non-numeric value", () => { + expect(extractRowCount([])).toBe(0) + expect(extractRowCount(undefined)).toBe(0) + expect(extractRowCount([{ _PROFILE_COUNT: "not-a-number" }])).toBe(0) + }) +}) diff --git a/packages/opencode/test/altimate/dbt-adapter-access.test.ts b/packages/opencode/test/altimate/dbt-adapter-access.test.ts index b565e5a58a..1c1f380a70 100644 --- a/packages/opencode/test/altimate/dbt-adapter-access.test.ts +++ b/packages/opencode/test/altimate/dbt-adapter-access.test.ts @@ -83,6 +83,8 @@ describe("feature tools in the allow-list", () => { test("each placeholder answers that it isn't implemented, as a failure", async () => { for (const id of FEATURE_TOOLS) { + // Tested in its own file once implemented. + if (id === "dbt_cte_profile") continue const tool = await initTool(EXPOSED[id] as any) expect(tool.id).toBe(id) const result = await tool.execute(MINIMAL_ARGS[id], ctx) diff --git a/packages/opencode/test/altimate/dbt-cte-profile.test.ts b/packages/opencode/test/altimate/dbt-cte-profile.test.ts new file mode 100644 index 0000000000..28ce9eeef0 --- /dev/null +++ b/packages/opencode/test/altimate/dbt-cte-profile.test.ts @@ -0,0 +1,202 @@ +// dbt_cte_profile: the CTE list, and one timed count query per CTE. The dbt adapter is replaced through +// `dbtAdapterInternals`, so no warehouse, Python or dbt is needed. +import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, test } from "bun:test" +import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises" +import os from "node:os" +import path from "node:path" +import { Effect } from "effect" +import type { Tool } from "@/tool/tool" +import { SessionID, MessageID } from "@/session/schema" +import { dbtAdapterInternals, resetDbtAdapter } from "@/altimate/native/connections/register" +import { DbtCteProfileTool } from "@/altimate/tools/dbt-cte-profile" +import { initTool } from "./tool-fixture" + +const ctx: Tool.Context = { + sessionID: SessionID.make("ses_dbt_cte_profile"), + messageID: MessageID.make("msg_dbt_cte_profile"), + agent: "analyst", + abort: new AbortController().signal, + messages: [], + metadata: () => Effect.void, + ask: () => Effect.void, +} + +const CUSTOMERS = `{{ config(materialized='table') }} + +-- orders first +with orders as ( + select * from {{ ref('stg_orders') }} +), + +"Big Spenders" (customer_id, total) as ( + select customer_id, sum(amount) from orders group by 1 +), + +final as ( + select * from "Big Spenders" +) + +select * from final +` + +let project: string +let queries: { sql: string; name: string; limit: number }[] +let respond: (sql: string) => Promise + +function useAdapter() { + dbtAdapterInternals.readConfig = async () => ({ + projectRoot: project, + pythonPath: "python3", + dbtIntegration: "corecommand", + queryLimit: 500, + }) + dbtAdapterInternals.create = async () => ({ + isInitialized: () => true, + dispose: async () => {}, + immediatelyExecuteSQLWithLimit: async (sql: string, name: string, limit: number) => { + queries.push({ sql, name, limit }) + return respond(sql) + }, + }) +} + +beforeAll(async () => { + project = await mkdtemp(path.join(os.tmpdir(), "dbt-cte-profile-")) + await writeFile(path.join(project, "dbt_project.yml"), "name: jaffle_shop\n") + await mkdir(path.join(project, "models", "marts"), { recursive: true }) + await mkdir(path.join(project, "models", "staging"), { recursive: true }) + await mkdir(path.join(project, "dbt_packages", "pkg", "models"), { recursive: true }) + await writeFile(path.join(project, "models", "marts", "customers.sql"), CUSTOMERS) + await writeFile(path.join(project, "models", "staging", "plain.sql"), "select 1 as id\n") + await writeFile(path.join(project, "models", "marts", "dup.sql"), "with a as (select 1) select * from a") + await writeFile(path.join(project, "models", "staging", "dup.sql"), "with b as (select 1) select * from b") + await writeFile(path.join(project, "dbt_packages", "pkg", "models", "ignored.sql"), "with z as (select 1) select * from z") +}) + +afterAll(async () => { + await rm(project, { recursive: true, force: true }) +}) + +beforeEach(() => { + queries = [] + respond = async () => ({ columnNames: ["_PROFILE_COUNT"], data: [{ _PROFILE_COUNT: 42 }] }) + resetDbtAdapter() + useAdapter() +}) + +afterEach(() => { + resetDbtAdapter() + delete dbtAdapterInternals.readConfig + delete dbtAdapterInternals.create +}) + +describe("dbt_cte_profile list", () => { + test("lists the CTEs in source order with their lines, without the adapter, on a read-only call", async () => { + const tool = await initTool(DbtCteProfileTool) + const result = await tool.execute({ model: "customers" }, { ...ctx, extra: { forceReadOnly: true } }) + expect(result.metadata.success).toBe(true) + expect(result.metadata.file).toBe(path.join("models", "marts", "customers.sql")) + expect(result.metadata.ctes).toEqual([ + { name: "orders", line: 4, index: 0, clause: expect.any(Number) }, + { name: '"Big Spenders"', line: 8, index: 1, clause: expect.any(Number) }, + { name: "final", line: 12, index: 2, clause: expect.any(Number) }, + ]) + expect(result.output).toContain("3 CTEs") + expect(queries).toEqual([]) + }) + + test("a model without CTEs lists none", async () => { + const tool = await initTool(DbtCteProfileTool) + const result = await tool.execute({ model: "plain" }, ctx) + expect(result.metadata.success).toBe(true) + expect(result.metadata.ctes).toEqual([]) + }) + + test("takes a project-relative path and ignores dbt_packages", async () => { + const tool = await initTool(DbtCteProfileTool) + const byPath = await tool.execute({ model: "models/marts/customers.sql" }, ctx) + expect(byPath.metadata.success).toBe(true) + const packaged = await tool.execute({ model: "ignored" }, ctx) + expect(packaged.metadata.success).toBe(false) + expect(packaged.output).toContain("no model named ignored") + }) + + test("an ambiguous name asks for the path; a path outside the project is refused", async () => { + const tool = await initTool(DbtCteProfileTool) + const dup = await tool.execute({ model: "dup" }, ctx) + expect(dup.metadata.success).toBe(false) + expect(dup.output).toContain("matches 2 files") + const outside = await tool.execute({ model: "../../etc/passwd.sql" }, ctx) + expect(outside.metadata.success).toBe(false) + expect(outside.output).toContain("outside the dbt project") + }) +}) + +describe("dbt_cte_profile run", () => { + test("runs one count query over the CTE and the earlier ones, limit 1, and reads the count by position", async () => { + const tool = await initTool(DbtCteProfileTool) + const result = await tool.execute({ model: "customers", cte: "Big Spenders" }, ctx) + expect(result.metadata.success).toBe(true) + expect(result.metadata.cte).toBe('"Big Spenders"') + expect(result.metadata.rows).toBe(42) + expect(typeof result.metadata.queryMs).toBe("number") + expect(queries).toHaveLength(1) + expect(queries[0].name).toBe('cte_profiler_"Big Spenders"') + expect(queries[0].limit).toBe(1) + expect(queries[0].sql).toContain("{{ config(materialized='table') }}") + expect(queries[0].sql).toContain("orders as (") + expect(queries[0].sql).toContain('"Big Spenders" (customer_id, total) as (') + expect(queries[0].sql).not.toContain("final as (") + expect(queries[0].sql).toEndWith('SELECT COUNT(*) AS _profile_count FROM "Big Spenders"') + }) + + test("matches the CTE name ignoring case and quotes", async () => { + const tool = await initTool(DbtCteProfileTool) + const result = await tool.execute({ model: "customers", cte: "ORDERS" }, ctx) + expect(result.metadata.success).toBe(true) + expect(result.metadata.cte).toBe("orders") + expect(queries[0].sql).toEndWith("FROM orders") + }) + + test("an unknown CTE names the ones the model has and runs nothing", async () => { + const tool = await initTool(DbtCteProfileTool) + const result = await tool.execute({ model: "customers", cte: "nope" }, ctx) + expect(result.metadata.success).toBe(false) + expect(result.output).toContain("orders") + expect(queries).toEqual([]) + }) + + test("a read-only call is refused and never starts the adapter", async () => { + const tool = await initTool(DbtCteProfileTool) + const result = await tool.execute({ model: "customers", cte: "orders" }, { ...ctx, extra: { forceReadOnly: true } }) + expect(result.metadata.success).toBe(false) + expect(result.metadata.refusal).toBe("read_only") + expect(result.output).toContain("--allow-write") + expect(queries).toEqual([]) + }) + + test("a failing query is reported as a failure, not thrown", async () => { + respond = async () => { + throw new Error("Object 'STG_ORDERS' does not exist") + } + const tool = await initTool(DbtCteProfileTool) + const result = await tool.execute({ model: "customers", cte: "orders" }, ctx) + expect(result.metadata.success).toBe(false) + expect(result.output).toContain("STG_ORDERS") + }) + + test("an empty result counts as zero rows", async () => { + respond = async () => ({ columnNames: ["_PROFILE_COUNT"], data: [] }) + const tool = await initTool(DbtCteProfileTool) + const result = await tool.execute({ model: "customers", cte: "final" }, ctx) + expect(result.metadata.rows).toBe(0) + }) + + test("a cancelled call throws instead of answering", async () => { + const controller = new AbortController() + controller.abort() + const tool = await initTool(DbtCteProfileTool) + await expect(tool.execute({ model: "customers", cte: "orders" }, { ...ctx, abort: controller.signal })).rejects.toBeDefined() + expect(queries).toEqual([]) + }) +}) From 21d0494cfbb1c87a2b1b3ba6e347f4f1be149856 Mon Sep 17 00:00:00 2001 From: Harish Kumar Date: Fri, 13 Nov 2026 17:01:49 +0530 Subject: [PATCH 03/26] fix: harden the dbt Power User tool foundation Co-Authored-By: Claude Opus 5.5 --- packages/opencode/src/altimate/api/client.ts | 4 +- .../src/altimate/tools/dbt-adapter-access.ts | 3 +- .../src/altimate/tools/dbt-cost-estimate.ts | 2 +- .../src/altimate/tools/dbt-cte-profile.ts | 2 +- .../src/altimate/tools/dbt-defer-state.ts | 2 +- .../src/altimate/tools/dbt-docs-coverage.ts | 10 ++-- .../opencode/src/altimate/tools/dbt-health.ts | 2 +- .../src/altimate/tools/dbt-map-relations.ts | 7 ++- .../src/altimate/tools/dbt-run-results.ts | 7 ++- .../src/altimate/tools/dbt-sql-graph.ts | 2 +- .../src/altimate/tools/docs-share-comments.ts | 2 +- .../test/altimate/dbt-adapter-access.test.ts | 52 ++++++++++++++++++- 12 files changed, 78 insertions(+), 17 deletions(-) diff --git a/packages/opencode/src/altimate/api/client.ts b/packages/opencode/src/altimate/api/client.ts index 069a182b7a..cb53c0ec7b 100644 --- a/packages/opencode/src/altimate/api/client.ts +++ b/packages/opencode/src/altimate/api/client.ts @@ -359,7 +359,9 @@ export namespace AltimateApi { } catch (e) { const p = credentialsPath() if (!(await Filesystem.exists(p))) - throw new Error(`No Altimate credentials are stored: sign in to Altimate in altimate-code first (they are saved to ${p}).`) + throw new Error( + `No Altimate credentials are stored: sign in to Altimate in altimate-code first (they are saved to ${p}).`, + ) throw new Error(`The Altimate credentials in ${p} can't be used: ${e instanceof Error ? e.message : String(e)}`) } return request(creds, method, endpoint, body) diff --git a/packages/opencode/src/altimate/tools/dbt-adapter-access.ts b/packages/opencode/src/altimate/tools/dbt-adapter-access.ts index a2711e35d1..d73dd7e9c5 100644 --- a/packages/opencode/src/altimate/tools/dbt-adapter-access.ts +++ b/packages/opencode/src/altimate/tools/dbt-adapter-access.ts @@ -67,7 +67,8 @@ export async function dbtAdapterFor(ctx: Pick { expect(tool.id).toBe(id) const result = await tool.execute(MINIMAL_ARGS[id], ctx) expect(result.title).toBe(`${id}: not implemented`) - expect(result.metadata.success).toBe(false) + expect(result.metadata).toEqual({ success: false, error: "not implemented", truncated: false }) expect(result.output).toContain(id) } }) @@ -149,6 +149,30 @@ describe("dbtAdapterFor", () => { } }) + test("says the adapter didn't start when the project exists but creating it fails", async () => { + dbtAdapterInternals.create = async () => { + created += 1 + throw new Error("dbt-integration missing") + } + const access = await dbtAdapterFor(ctx) + expect(access).toEqual({ refusal: "not_started", reason: "the dbt adapter didn't start" }) + expect(created).toBe(1) + }) + + test("a call cancelled while the adapter starts throws and stops waiting", async () => { + let entered = () => {} + const starting = new Promise((resolve) => (entered = resolve)) + dbtAdapterInternals.create = () => { + entered() + return new Promise(() => {}) + } + const controller = new AbortController() + const pending = dbtAdapterFor({ ...ctx, abort: controller.signal }) + await starting + controller.abort(new Error("cancelled")) + await expect(pending).rejects.toThrow("cancelled") + }) + test("a cancelled call throws rather than answering", async () => { const controller = new AbortController() controller.abort() @@ -169,6 +193,28 @@ describe("dbtAdapterFor", () => { await disposeInstance(project) } }) + + test("a session in another dbt project than the adapter's says which", async () => { + const other = await mkdtemp(path.join(os.tmpdir(), "dbt-adapter-other-")) + await writeFile(path.join(other, "dbt_project.yml"), "name: other\n") + dbtAdapterInternals.readConfig = async () => ({ + projectRoot: other, + pythonPath: "python3", + dbtIntegration: "corecommand", + queryLimit: 500, + }) + try { + const access = await Instance.provide({ directory: project, fn: () => dbtAdapterFor(ctx) }) + expect(access).toEqual({ + refusal: "other_project", + reason: `the dbt adapter is set up for ${realpathSync(other)}, not ${realpathSync(project)}`, + }) + expect(created).toBe(0) + } finally { + await disposeInstance(project) + await rm(other, { recursive: true, force: true }) + } + }) }) describe("AltimateApi.apiRequest", () => { @@ -216,6 +262,8 @@ describe("AltimateApi.apiRequest", () => { test("names the file when the stored credentials can't be read", async () => { await Bun.write(path.join(home, ".altimate", "altimate.json"), "{not json") - await expect(AltimateApi.apiRequest("GET", "/dbt/v1/x")).rejects.toThrow("can't be used") + await expect(AltimateApi.apiRequest("GET", "/dbt/v1/x")).rejects.toThrow( + `The Altimate credentials in ${path.join(home, ".altimate", "altimate.json")} can't be used`, + ) }) }) From 86e87465c28f4587eee9328d9f726901179f37b2 Mon Sep 17 00:00:00 2001 From: Harish Kumar Date: Fri, 13 Nov 2026 17:27:50 +0530 Subject: [PATCH 04/26] fix: dbt_cte_profile reads a model's own CTEs only, names each one by position, and reports failures as errors The scanner now parses the top-level WITH clause in order, so strings that hold comment markers or CTE-like text, doubled quotes in quoted names, a WITH inside a CTE body, and large files no longer misdetect or stall it. A CTE is chosen by name or by position (a name that matches several is refused), the list is read from the session's project, a unique_id resolves only through the manifest, only .sql files are read, failures carry an error so the CLI and MCP report them, and the count query runs under the call's abort and timeout. Co-Authored-By: Claude Sonnet 5.5 --- .../opencode/src/altimate/tools/cte-detect.ts | 395 ++++++------------ .../src/altimate/tools/dbt-cte-profile.ts | 101 +++-- .../opencode/test/altimate/cte-detect.test.ts | 72 +++- .../test/altimate/dbt-adapter-access.test.ts | 2 +- .../test/altimate/dbt-cte-profile.test.ts | 165 +++++++- 5 files changed, 402 insertions(+), 333 deletions(-) diff --git a/packages/opencode/src/altimate/tools/cte-detect.ts b/packages/opencode/src/altimate/tools/cte-detect.ts index 33b64b6a08..a4eea27eea 100644 --- a/packages/opencode/src/altimate/tools/cte-detect.ts +++ b/packages/opencode/src/altimate/tools/cte-detect.ts @@ -23,324 +23,168 @@ export interface CteInfo { withClauseStart: number } -export interface CteProfileEntry { - name: string - line: number - queryTimeMs: number - marginalTimeMs: number - rowCount: number - tier: "hot" | "warm" | "cool" -} - -/** Longest quoted identifier ("name", `name`, [name]) the identifier regex accepts. */ -const MAX_QUOTED_IDENTIFIER_LENGTH = 500 - -/** Longest CTE column list the identifier regex accepts. */ -const MAX_COLUMN_LIST_LENGTH = 1000 - -const IDENT_PART = - `(?:[a-zA-Z_][a-zA-Z0-9_]*|"[^"]{1,${MAX_QUOTED_IDENTIFIER_LENGTH}}"|` + - `\`[^\`]{1,${MAX_QUOTED_IDENTIFIER_LENGTH}}\`|` + - `\\[[^\\]]{1,${MAX_QUOTED_IDENTIFIER_LENGTH}}\\])` - -function identifierRegex(): RegExp { - return new RegExp(`(${IDENT_PART}(?:\\.${IDENT_PART})*(?:\\s*\\([^)]{0,${MAX_COLUMN_LIST_LENGTH}}\\))?)`, "gi") -} - -interface StringState { - newPos: number - inString: boolean - stringChar: string -} - -function handleSqlStringLiteral(text: string, pos: number, inString: boolean, stringChar: string): StringState { - const char = text[pos] - const nextChar = pos < text.length - 1 ? text[pos + 1] : "" - - if (!inString && (char === "'" || char === '"')) { - return { newPos: pos, inString: true, stringChar: char } - } - if (inString && char === stringChar) { - if (nextChar === stringChar) { - // Doubled quote is an escaped quote: skip the second one. - return { newPos: pos + 1, inString: true, stringChar } - } - return { newPos: pos, inString: false, stringChar: "" } - } - return { newPos: pos, inString, stringChar } -} - -/** - * When `pos` starts a comment (`--`, block, or Jinja), return the position of - * its last character (the newline for line comments); otherwise return `pos`. - */ -function handleSqlComment(text: string, pos: number): number { +/** Position just past the string literal, quoted identifier or comment (`--`, block, Jinja) that starts at `pos`; `pos` itself when none does. An unterminated one runs to the end of the text. */ +function skipLiteral(text: string, pos: number): number { const char = text[pos] - const nextChar = pos < text.length - 1 ? text[pos + 1] : "" - - if (char === "-" && nextChar === "-") { - let endPos = pos + 2 - while (endPos < text.length && text[endPos] !== "\n" && text[endPos] !== "\r") endPos++ - return endPos - } - - if (char === "/" && nextChar === "*") { - let endPos = pos + 2 - while (endPos < text.length - 1) { - if (text[endPos] === "*" && text[endPos + 1] === "/") return endPos + 1 - endPos++ - } - return text.length - } - - if (char === "{" && nextChar === "#") { - let endPos = pos + 2 - while (endPos < text.length - 1) { - if (text[endPos] === "#" && text[endPos + 1] === "}") return endPos + 1 - endPos++ + const next = text[pos + 1] + if (char === "'" || char === '"' || char === "`") { + let i = pos + 1 + while (i < text.length) { + if (text[i] === char) { + // A doubled quote is an escaped quote. + if (text[i + 1] === char) i += 2 + else return i + 1 + } else i++ } return text.length } - - return pos + const close = char === "-" && next === "-" ? "\n" : char === "/" && next === "*" ? "*/" : char === "{" && next === "#" ? "#}" : null + if (close === null) return pos + const end = text.indexOf(close, pos + 2) + if (end === -1) return text.length + return close === "\n" ? end : end + close.length } -function isPositionInsideComment(content: string, position: number): boolean { - let pos = 0 - let inBlockComment = false - let inJinjaComment = false - - while (pos < position && pos < content.length) { - const char = content[pos] - const nextChar = pos < content.length - 1 ? content[pos + 1] : "" - - if (!inBlockComment && !inJinjaComment && char === "/" && nextChar === "*") { - inBlockComment = true - pos += 2 - continue - } - if (inBlockComment && char === "*" && nextChar === "/") { - inBlockComment = false - pos += 2 - continue - } - if (!inBlockComment && !inJinjaComment && char === "{" && nextChar === "#") { - inJinjaComment = true - pos += 2 +/** Position of the first character at or after `pos` that is not whitespace, a comment, or (when `jinja`) a Jinja `{% %}` tag. */ +function skipTrivia(text: string, pos: number, jinja: boolean): number { + while (pos < text.length) { + if (/\s/.test(text[pos])) { + pos++ continue } - if (inJinjaComment && char === "#" && nextChar === "}") { - inJinjaComment = false - pos += 2 + const end = skipLiteral(text, pos) + if (end !== pos && !/['"`]/.test(text[pos])) { + pos = end continue } - if (!inBlockComment && !inJinjaComment && char === "-" && nextChar === "-") { - const lineStart = pos - let lineEnd = pos + 2 - while (lineEnd < content.length && content[lineEnd] !== "\n" && content[lineEnd] !== "\r") lineEnd++ - if (position >= lineStart && position < lineEnd) return true - pos = lineEnd + if (jinja && text[pos] === "{" && text[pos + 1] === "%") { + const close = text.indexOf("%}", pos + 2) + if (close === -1) return text.length + pos = close + 2 continue } - pos++ + return pos } - - return inBlockComment || inJinjaComment -} - -/** Every WITH keyword outside comments and strings. */ -function findWithKeywords(text: string): number[] { - const positions: number[] = [] - let pos = 0 - let inString = false - let stringChar = "" - - while (pos < text.length) { - const commentEnd = handleSqlComment(text, pos) - if (commentEnd !== pos) { - pos = commentEnd + 1 - continue - } - - const s = handleSqlStringLiteral(text, pos, inString, stringChar) - pos = s.newPos - inString = s.inString - stringChar = s.stringChar - - if (!inString) { - const m = text.substring(pos).match(/^with\b/i) - if (m) { - const before = pos > 0 ? text[pos - 1] : " " - if (!/[a-zA-Z0-9_]/.test(before)) positions.push(pos) - pos += m[0].length - continue - } - } - pos++ - } - return positions + return pos } +/** Offset of the `)` that closes the `(` at `openParenPos`, or -1 when it is never closed. */ function findMatchingClosingParen(text: string, openParenPos: number): number { - let parenCount = 1 + let depth = 1 let pos = openParenPos + 1 - let inString = false - let stringChar = "" - - while (pos < text.length && parenCount > 0) { - const char = text[pos] - - const commentEnd = handleSqlComment(text, pos) - if (commentEnd !== pos) { - pos = commentEnd + while (pos < text.length) { + const end = skipLiteral(text, pos) + if (end !== pos) { + pos = end continue } - - const s = handleSqlStringLiteral(text, pos, inString, stringChar) - pos = s.newPos - inString = s.inString - stringChar = s.stringChar - - if (!inString) { - if (char === "(") parenCount++ - else if (char === ")") parenCount-- - } + if (text[pos] === "(") depth++ + else if (text[pos] === ")" && --depth === 0) return pos pos++ } - - return parenCount === 0 ? pos - 1 : -1 + return -1 } -/** Position of the main SELECT after the CTEs, the end of text, or -1 when a nested WITH is found. */ -function findWithClauseEnd(text: string, withStartPos: number): number { - let pos = withStartPos - let parenCount = 0 - let inString = false - let stringChar = "" - +/** Every WITH keyword outside comments, strings and parentheses: a WITH inside a CTE body or a subquery belongs to that query, not to the model. */ +function findTopLevelWithKeywords(text: string): number[] { + const positions: number[] = [] + const keyword = /with\b/iy + let depth = 0 + let pos = 0 while (pos < text.length) { - const char = text[pos] - - const commentEnd = handleSqlComment(text, pos) - if (commentEnd !== pos) { - pos = commentEnd + const end = skipLiteral(text, pos) + if (end !== pos) { + pos = end continue } - - const s = handleSqlStringLiteral(text, pos, inString, stringChar) - pos = s.newPos - inString = s.inString - stringChar = s.stringChar - - if (!inString) { - if (char === "(") { - parenCount++ - } else if (char === ")") { - parenCount-- - } else if (parenCount === 0) { - const rest = text.substring(pos) - if (/^\s*with\b/i.test(rest)) return -1 - if (/^\s*select\b/i.test(rest)) return pos + const char = text[pos] + if (char === "(") depth++ + else if (char === ")") depth = Math.max(0, depth - 1) + else if (depth === 0 && (char === "w" || char === "W") && !/[A-Za-z0-9_$]/.test(text[pos - 1] ?? " ")) { + keyword.lastIndex = pos + if (keyword.test(text)) { + positions.push(pos) + pos += 4 + continue } } pos++ } - return text.length -} - -/** `identifier [(columns)] [comments] AS (` starting at `startIndex`; returns the matched text or undefined. */ -function validateCteMatchWithComments(content: string, startIndex: number, identifierName: string): string | undefined { - let pos = startIndex + identifierName.length - - while (pos < content.length && /\s/.test(content[pos])) pos++ - if (pos < content.length && content[pos] === "(") { - const end = findMatchingClosingParen(content, pos) - if (end !== -1) pos = end + 1 - } - - while (pos < content.length) { - while (pos < content.length && /\s/.test(content[pos])) pos++ - if (pos >= content.length) break - - const commentEnd = handleSqlComment(content, pos) - if (commentEnd !== pos) { - pos = commentEnd + 1 - continue - } - - const asMatch = content.substring(pos).match(/^as\s*\(/i) - if (asMatch) return content.substring(startIndex, pos + asMatch[0].length) - break - } - return undefined + return positions } -function findCteIdentifiersOnly(withClauseContent: string): Array<{ index: number; name: string; fullMatch: string }> { - const regex = identifierRegex() - const out: Array<{ index: number; name: string; fullMatch: string }> = [] - let match: RegExpExecArray | null - - while ((match = regex.exec(withClauseContent)) !== null) { - const name = match[1] - if (isPositionInsideComment(withClauseContent, match.index)) continue - const fullMatch = validateCteMatchWithComments(withClauseContent, match.index, name) - if (fullMatch !== undefined) out.push({ index: match.index, name, fullMatch }) +/** Offset just past the identifier (`a`, `"a""b"`, `` `a` ``, `[a]`, `db.schema.a`) that starts at `pos`; -1 when none does. */ +function readIdentifier(text: string, pos: number): number { + let end = pos + for (;;) { + const char = text[end] + if (char === '"' || char === "`") { + const close = skipLiteral(text, end) + if (close === end || text[close - 1] !== char || close - end < 2) return -1 + end = close + } else if (char === "[") { + const close = text.indexOf("]", end + 1) + if (close === -1) return -1 + end = close + 1 + } else { + const bare = /[A-Za-z_][A-Za-z0-9_]*/y + bare.lastIndex = end + if (!bare.test(text)) return -1 + end = bare.lastIndex + } + if (text[end] !== ".") return end + end++ } - return out } -function extractCtesFromWithClause( - withClauseContent: string, - withStartPos: number, - withClauseStart: number, - ctes: CteInfo[], -): void { - let cteIndex = 0 - - for (const m of findCteIdentifiersOnly(withClauseContent)) { - const nameStart = withStartPos + m.index - const nameEnd = nameStart + m.name.length - const openParenRel = m.index + m.fullMatch.length - 1 - const closeRel = findMatchingClosingParen(withClauseContent, openParenRel) - if (closeRel === -1) continue - +/** The CTEs of the WITH clause that opens at `withPos`: each `name [(columns)] AS (body)`, separated by commas, up to the first thing that is not one. */ +function parseWithClause(text: string, withPos: number, ctes: CteInfo[]): void { + let pos = skipTrivia(text, withPos + 4, true) + const recursive = /recursive\b/iy + recursive.lastIndex = pos + if (recursive.test(text)) pos = skipTrivia(text, pos + "recursive".length, true) + + const asOpen = /as(?:\s+(?:not\s+)?materialized)?\s*\(/iy + for (let index = 0; ; index++) { + const nameStart = pos + const identifierEnd = readIdentifier(text, nameStart) + if (identifierEnd === -1) return + let nameEnd = identifierEnd + let name = text.substring(nameStart, identifierEnd) + pos = skipTrivia(text, identifierEnd, false) + if (text[pos] === "(") { + const close = findMatchingClosingParen(text, pos) + if (close === -1) return + // The name as written plus its column list; a comment between them is left out. + name += ` ${text.substring(pos, close + 1)}` + nameEnd = close + 1 + pos = skipTrivia(text, nameEnd, false) + } + asOpen.lastIndex = pos + const header = asOpen.exec(text) + if (!header) return + const open = pos + header[0].length - 1 + const close = findMatchingClosingParen(text, open) + if (close === -1) return ctes.push({ - name: m.name, + name, nameStart, nameEnd, - queryStart: withStartPos + openParenRel + 1, - queryEnd: withStartPos + closeRel, - index: cteIndex, - withClauseStart, + queryStart: open + 1, + queryEnd: close, + index, + withClauseStart: withPos, }) - cteIndex++ + pos = skipTrivia(text, close + 1, true) + if (text[pos] !== ",") return + pos = skipTrivia(text, pos + 1, true) } } -/** Find every CTE in the model text, in source order. */ +/** Find every CTE of the model's own WITH clauses, in source order. A WITH inside a CTE body or a subquery is not listed: its CTEs cannot be counted without the query around them. */ export function detectCtes(text: string): CteInfo[] { const ctes: CteInfo[] = [] - - for (const withPos of findWithKeywords(text)) { - let start = withPos + 4 // "with".length - - while (start < text.length) { - while (start < text.length && /\s/.test(text[start])) start++ - if (start >= text.length) break - const commentEnd = handleSqlComment(text, start) - if (commentEnd !== start) { - start = commentEnd + 1 - continue - } - break - } - - const end = findWithClauseEnd(text, start) - if (end === -1) continue - - extractCtesFromWithClause(text.substring(start, end), start, withPos, ctes) - } - + for (const withPos of findTopLevelWithKeywords(text)) parseWithClause(text, withPos, ctes) return ctes } @@ -358,8 +202,11 @@ export function quoteSqlIdentifier(identifier: string): string { /** * `SELECT COUNT(*)` over one CTE, keeping the preamble (dbt configs, variables) * and every earlier CTE of the same WITH clause, so the query compiles on its own. + * Undefined for a CTE of a second WITH clause in the same file, which has no standalone query. */ export function buildCountQuery(text: string, ctes: CteInfo[], target: CteInfo): string | undefined { + // The text before a later WITH holds the earlier clause's whole query: nothing valid stands alone there. + if (ctes.some((c) => c.withClauseStart < target.withClauseStart)) return undefined const upToTarget = ctes.filter((c) => c.withClauseStart === target.withClauseStart && c.index <= target.index) const definitions: string[] = [] diff --git a/packages/opencode/src/altimate/tools/dbt-cte-profile.ts b/packages/opencode/src/altimate/tools/dbt-cte-profile.ts index 8d085c0ed5..09c0ae0f1f 100644 --- a/packages/opencode/src/altimate/tools/dbt-cte-profile.ts +++ b/packages/opencode/src/altimate/tools/dbt-cte-profile.ts @@ -4,6 +4,7 @@ import z from "zod" import { Tool } from "../../tool/tool" import { loadRawManifest, findModel } from "../native/dbt/helpers" import { projectRoot } from "../native/connections/registry" +import { executeQuery } from "../native/query-execution" import { dbtAdapterFor } from "./dbt-adapter-access" import { buildCountQuery, detectCtes, extractRowCount, lineOf, stripColumnList, type CteInfo } from "./cte-detect" @@ -16,11 +17,13 @@ const MAX_SEARCH_DEPTH = 12 /** Largest model file the tool reads. */ const MAX_MODEL_BYTES = 4 * 1024 * 1024 -/** A CTE name without its column list, quotes or letter case, for comparing what the caller typed. */ -function nameKey(name: string): string { - return stripColumnList(name) - .replace(/^["'`[]|["'`\]]$/g, "") - .toLowerCase() +/** A CTE name without its column list or surrounding quotes, quote escapes undone. */ +function bareName(name: string): string { + const plain = stripColumnList(name) + const quote = plain[0] + if ((quote === '"' || quote === "`") && plain.length > 1 && plain.endsWith(quote)) return plain.slice(1, -1).replaceAll(quote + quote, quote) + if (quote === "[" && plain.endsWith("]")) return plain.slice(1, -1) + return plain } function inside(root: string, file: string): boolean { @@ -60,52 +63,61 @@ function findSqlFiles(root: string, name: string): string[] { return found.sort() } -/** The model's source file: a project-relative path, the manifest's record of it, or a search by name. */ +/** The model's source file: a project-relative or absolute `.sql` path, the manifest's record of a name or unique_id, or a search by name. */ function modelFile(root: string, model: string): { file?: string; error?: string } { const realRoot = realpath(root) if (/\.sql$/i.test(model) || /[\\/]/.test(model)) { const candidate = path.resolve(realRoot, model) + if (!/\.sql$/i.test(candidate)) return { error: `${model} is not a .sql file` } if (!inside(realRoot, realpath(candidate))) return { error: `${model} is outside the dbt project` } - return fs.existsSync(candidate) ? { file: candidate } : { error: `no file ${model} in the dbt project` } + return fs.existsSync(candidate) && fs.statSync(candidate).isFile() ? { file: realpath(candidate) } : { error: `no file ${model} in the dbt project` } } try { const manifest = loadRawManifest(path.join(realRoot, "target", "manifest.json")) const node = manifest?.nodes ? findModel(manifest.nodes, model) : null if (node?.original_file_path) { const candidate = path.resolve(realRoot, node.original_file_path) - if (inside(realRoot, realpath(candidate)) && fs.existsSync(candidate)) return { file: candidate } + if (inside(realRoot, realpath(candidate)) && fs.existsSync(candidate)) return { file: realpath(candidate) } } } catch { // A manifest that can't be read doesn't stop the search by name. } - const name = model.startsWith("model.") ? (model.split(".").pop() ?? model) : model - const matches = findSqlFiles(realRoot, name) - if (matches.length === 1) return { file: matches[0] } + // A unique_id names one model of one package: only the manifest can say which file it is. + if (model.startsWith("model.")) return { error: `${model} is not in target/manifest.json: run dbt compile, or pass the model's name or path` } + const matches = findSqlFiles(realRoot, model) + if (matches.length === 1) return { file: realpath(matches[0]) } if (matches.length === 0) return { error: `no model named ${model} in the dbt project` } const rel = matches.map((m) => path.relative(realRoot, m)) return { error: `${model} matches ${matches.length} files (${rel.join(", ")}): pass the path` } } +/** The dbt project of the session's directory; else the one the dbt adapter is set up for. */ async function projectDir(): Promise { + const { findProjectRoot } = await import("../../../../dbt-tools/src/config") + const here = findProjectRoot(projectRoot()) + if (here) return here const { getDbtProjectRoot } = await import("../native/connections/register") - const root = await getDbtProjectRoot() - if (root) return root - const here = projectRoot() - return fs.existsSync(path.join(here, "dbt_project.yml")) ? here : undefined + return getDbtProjectRoot() } function failure(title: string, output: string, extra: Record = {}) { - return { title: `dbt_cte_profile: ${title}`, metadata: { success: false, ...extra }, output } + return { title: `dbt_cte_profile: ${title}`, metadata: { success: false, error: output, ...extra }, output } } export const DbtCteProfileTool = Tool.define("dbt_cte_profile", { description: - "List a dbt model's CTEs, or time one CTE with a row-count query. Without `cte` it returns the CTE names in source order and runs nothing (a read-only call works). With `cte` it runs `SELECT COUNT(*)` over that CTE and the CTEs before it in the same WITH clause on the warehouse, and returns the elapsed milliseconds and the row count; that run needs write permission (`--allow-write`) because it starts the dbt adapter and compiles the model. Each run recomputes the earlier CTEs, so a model of N CTEs costs N queries.", + "List a dbt model's CTEs, or time one CTE with a row-count query. Without `cte` or `position` it returns the CTE names in source order (name, line, position) and runs nothing, so a read-only call works. With one of them it runs `SELECT COUNT(*)` over that CTE and the CTEs before it in the same WITH clause on the warehouse, and returns the elapsed milliseconds and the row count; that run needs write permission (`--allow-write`) because it starts the dbt adapter and compiles the model. Each run recomputes the earlier CTEs, so a model of N CTEs costs N queries. Only the model's own top-level WITH clause is read: a WITH inside a CTE body is part of that CTE.", parameters: z.object({ model: z .string() - .describe("Model name, unique_id, or project-relative path to the model's .sql file (e.g. 'customers' or 'models/marts/customers.sql')"), - cte: z.string().optional().describe("CTE to time; without it, the CTE list is returned"), + .describe("Model name, unique_id, or project-relative or absolute path to the model's .sql file (e.g. 'customers' or 'models/marts/customers.sql')"), + cte: z.string().optional().describe("CTE to time, by name; without it (and without `position`), the CTE list is returned"), + position: z + .number() + .int() + .min(0) + .optional() + .describe("The CTE's position in the list (0-based); needed when several CTEs share a name"), }), async execute(args, ctx) { const root = await projectDir() @@ -118,15 +130,16 @@ export const DbtCteProfileTool = Tool.define("dbt_cte_profile", { const ctes = detectCtes(text) const file = path.relative(realpath(root), located.file) - if (!args.cte) { - const list = ctes.map((c) => ({ + if (args.cte === undefined && args.position === undefined) { + const list = ctes.map((c, position) => ({ name: stripColumnList(c.name), line: lineOf(text, c.nameStart) + 1, index: c.index, clause: c.withClauseStart, + position, })) const output = list.length - ? [`${list.length} CTE${list.length === 1 ? "" : "s"} in ${file}:`, ...list.map((c, i) => `${i + 1}. ${c.name} (line ${c.line})`)].join("\n") + ? [`${list.length} CTE${list.length === 1 ? "" : "s"} in ${file}:`, ...list.map((c) => `${c.position + 1}. ${c.name} (line ${c.line})`)].join("\n") : `No CTEs in ${file}.` return { title: `dbt_cte_profile: ${list.length} CTE${list.length === 1 ? "" : "s"} in ${args.model}`, @@ -135,32 +148,46 @@ export const DbtCteProfileTool = Tool.define("dbt_cte_profile", { } } - const wanted = nameKey(args.cte) - const matches = ctes.filter((c) => nameKey(c.name) === wanted) - if (matches.length === 0) { - const names = ctes.map((c) => stripColumnList(c.name)).join(", ") - return failure("CTE not found", `${args.model} has no CTE named ${args.cte}${names ? `; it has: ${names}` : ""}.`, { model: args.model }) + const names = ctes.map((c) => stripColumnList(c.name)).join(", ") + let target: CteInfo | undefined + if (args.position !== undefined) { + target = ctes[args.position] + if (!target) return failure("CTE not found", `${args.model} has ${ctes.length} CTEs: there is none at position ${args.position}.`, { model: args.model }) + if (args.cte !== undefined && bareName(target.name) !== bareName(args.cte)) + return failure("CTE not found", `The CTE at position ${args.position} of ${args.model} is ${stripColumnList(target.name)}, not ${args.cte}.`, { model: args.model }) + } else { + // The name as typed wins; letter case only decides when nothing matches it exactly. + const wanted = bareName(args.cte ?? "") + const exact = ctes.filter((c) => bareName(c.name) === wanted) + const matches = exact.length > 0 ? exact : ctes.filter((c) => bareName(c.name).toLowerCase() === wanted.toLowerCase()) + if (matches.length === 0) + return failure("CTE not found", `${args.model} has no CTE named ${args.cte}${names ? `; it has: ${names}` : ""}.`, { model: args.model }) + if (matches.length > 1) { + const at = matches.map((c) => ctes.indexOf(c)).join(", ") + return failure("CTE ambiguous", `${args.model} has ${matches.length} CTEs that match ${args.cte} (positions ${at}): pass position.`, { model: args.model }) + } + target = matches[0] } - const target: CteInfo = matches[0] const query = buildCountQuery(text, ctes, target) - if (!query) return failure("no query", `Could not build a count query for ${args.cte}.`, { model: args.model, cte: args.cte }) + const name = stripColumnList(target.name) + if (!query) return failure("no query", `${name} is in a second WITH clause of ${args.model}, which has no query of its own to count.`, { model: args.model, cte: name }) const access = await dbtAdapterFor(ctx) - if (!access.adapter) { - return failure("not run", `${access.reason}`, { model: args.model, cte: args.cte, refusal: access.refusal }) - } + if (!access.adapter) return failure("not run", access.reason ?? "the dbt adapter is not available", { model: args.model, cte: name, refusal: access.refusal }) + const adapter = access.adapter ctx.abort.throwIfAborted() - const name = stripColumnList(target.name) const started = performance.now() try { - const result = await access.adapter.immediatelyExecuteSQLWithLimit(query, `cte_profiler_${name}`, 1) + // The adapter has no cancel of its own: the run's timeout, budget and abort end the wait, not the query. + const result: { data?: Record[] } = await executeQuery({ type: "dbt" } as any, {}, { signal: ctx.abort, sessionID: ctx.sessionID }, () => + adapter.immediatelyExecuteSQLWithLimit(query, `cte_profiler_${name}`, 1), + ) const queryMs = Math.round(performance.now() - started) const rows = extractRowCount(result?.data) - const note = matches.length > 1 ? ` (${matches.length} CTEs share this name; the first was timed)` : "" return { title: `dbt_cte_profile: ${name} ${queryMs} ms`, - metadata: { success: true, model: args.model, cte: name, index: target.index, queryMs, rows }, - output: `${name}: ${queryMs} ms, ${rows} row${rows === 1 ? "" : "s"}${note}`, + metadata: { success: true, model: args.model, cte: name, index: target.index, position: ctes.indexOf(target), queryMs, rows }, + output: `${name}: ${queryMs} ms, ${rows} row${rows === 1 ? "" : "s"}`, } } catch (e) { if (ctx.abort.aborted) throw e diff --git a/packages/opencode/test/altimate/cte-detect.test.ts b/packages/opencode/test/altimate/cte-detect.test.ts index 8f7b86c372..9bce1298b1 100644 --- a/packages/opencode/test/altimate/cte-detect.test.ts +++ b/packages/opencode/test/altimate/cte-detect.test.ts @@ -138,7 +138,6 @@ describe("detectCtes", () => { expect(ctes).toHaveLength(1) assertCTE(ctes[0], "source", 0, "CTE with single-line block comment between name and AS keyword") - // Skip content extraction test for now due to Range/Position mocking complexity // The important part is that CTE detection works correctly }) @@ -157,7 +156,6 @@ select * from source` expect(ctes).toHaveLength(1) assertCTE(ctes[0], "source", 0, "CTE with multi-line block comment between name and AS keyword") - // Skip content extraction test for now due to Range/Position mocking complexity // The important part is that CTE detection works correctly }) @@ -942,9 +940,8 @@ select * from renamed` const ctes = detectCtes(document) - // Current limitation: Comments between CTE name and column list break the parsing - // This is an edge case that would require more complex parsing logic - expect(ctes).toHaveLength(0) + expect(ctes).toHaveLength(1) + assertCTE(ctes[0], "my_cte (id, name)", 0) }) it("should handle comments after the closing parenthesis", () => { @@ -1270,7 +1267,6 @@ select * from renamed` expect(ctes).toHaveLength(1) assertCTE(ctes[0], "my_cte", 0, "CTE with correlated subquery counting from other table") - // Skip content extraction test - focus on CTE detection correctness }) it("should handle unterminated comments gracefully", () => { @@ -1378,15 +1374,59 @@ describe("detectCtes offsets", () => { expect(detectCtesInText(sql)).toEqual([]) }) - it("skips a WITH clause that contains another top-level WITH and keeps the inner one", () => { + it("lists the CTEs of each top-level WITH, each with its own clause", () => { const sql = "with a as (select 1) with b as (select 2) select * from b" - expect(detectCtesInText(sql).map((c) => c.name)).toEqual(["b"]) + const ctes = detectCtesInText(sql) + expect(ctes.map((c) => [c.name, c.index])).toEqual([["a", 0], ["b", 0]]) + expect(ctes[0].withClauseStart).not.toBe(ctes[1].withClauseStart) + }) + + it("lists only the model's own CTEs: a WITH inside a CTE body is part of that body", () => { + const sql = `with outer_cte as ( + with inner_cte as (select 1 as x) + select x from inner_cte +), +last as (select * from outer_cte) +select * from last` + const ctes = detectCtesInText(sql) + expect(ctes.map((c) => [c.name, c.index])).toEqual([["outer_cte", 0], ["last", 1]]) + // The outer CTE's count query carries its nested WITH whole and closes every parenthesis. + const query = buildCountQuery(sql, ctes, ctes[1])! + expect(query).toContain("inner_cte as (select 1 as x)") + expect(query.match(/\(/g)!.length).toBe(query.match(/\)/g)!.length) + expect(query).toEndWith("SELECT COUNT(*) AS _profile_count FROM last") + }) + + it("a WITH inside a subquery or a window definition is not a model CTE", () => { + const sql = "select * from (with s as (select 1) select * from s) t join u on true window w as (partition by a)" + expect(detectCtesInText(sql)).toEqual([]) + }) + + it("comment markers and CTE-like text inside string literals neither hide nor add CTEs", () => { + expect(detectCtesInText("with a as (select '--' as s), b as (select 2) select * from b").map((c) => c.name)).toEqual(["a", "b"]) + expect(detectCtesInText("with a as (select '/*' as s), b as (select '*/' as t) select * from b").map((c) => c.name)).toEqual(["a", "b"]) + expect(detectCtesInText("with a as (select '(x)' as s, 'fake as (select 2)' as t) select * from a").map((c) => c.name)).toEqual(["a"]) + expect(detectCtesInText("with a as (select 'it''s ) as (' as s), b as (select 2) select * from b").map((c) => c.name)).toEqual(["a", "b"]) + }) + + it("a quoted name keeps its escaped quote", () => { + const sql = 'with "a""b" as (select 1), `c``d` as (select 2) select * from "a""b"' + expect(detectCtesInText(sql).map((c) => c.name)).toEqual(['"a""b"', "`c``d`"]) + }) + + it("keeps CTEs that follow a Jinja tag between them, and reads WITH RECURSIVE and MATERIALIZED", () => { + const jinja = "with a as (select 1),\n{% if x %}\nb as (select 2),\n{% endif %}\nc as (select 3)\nselect * from c" + expect(detectCtesInText(jinja).map((c) => c.name)).toEqual(["a", "b", "c"]) + expect(detectCtesInText("with recursive r as (select 1) select * from r").map((c) => c.name)).toEqual(["r"]) + expect(detectCtesInText("with m as materialized (select 1), n as not materialized (select 2) select 1").map((c) => c.name)).toEqual(["m", "n"]) }) - it("tracks CTE of nested WITH inside a body without confusing the outer index", () => { - const sql = "with outer_cte as (with inner_cte as (select 1 as x) select x from inner_cte) select * from outer_cte" - const names = detectCtesInText(sql).map((c) => c.name) - expect(names).toContain("outer_cte") + it("scans in linear time: a 2 MB model is read in well under a second", () => { + const body = "select a, b, c from t where x = 1 and y = 'it''s' -- note\n" + const sql = `with big as (\n${body.repeat(Math.floor(2_000_000 / body.length))}) select * from big` + const started = performance.now() + expect(detectCtesInText(sql).map((c) => c.name)).toEqual(["big"]) + expect(performance.now() - started).toBeLessThan(1000) }) it("reports zero-based lines", () => { @@ -1433,13 +1473,11 @@ describe("buildCountQuery", () => { expect(query.endsWith("FROM a")).toBe(true) }) - it("only includes CTEs of the target's own WITH clause", () => { + it("builds no query for a CTE of a second WITH clause, whose preamble would hold the first clause's query", () => { const text = "with a as (select 1) select * from a;\nwith b as (select 2) select * from b" const ctes = detectCtesInText(text) - const second = ctes.find((c) => c.name === "b")! - const query = buildCountQuery(text, ctes, second)! - expect(query).not.toContain("WITH a") - expect(query).toContain("WITH b as (") + expect(buildCountQuery(text, ctes, ctes.find((c) => c.name === "a")!)).toContain("WITH a as (") + expect(buildCountQuery(text, ctes, ctes.find((c) => c.name === "b")!)).toBeUndefined() }) it("returns undefined for a CTE that is not in the list", () => { diff --git a/packages/opencode/test/altimate/dbt-adapter-access.test.ts b/packages/opencode/test/altimate/dbt-adapter-access.test.ts index 1c1f380a70..cb181eda67 100644 --- a/packages/opencode/test/altimate/dbt-adapter-access.test.ts +++ b/packages/opencode/test/altimate/dbt-adapter-access.test.ts @@ -83,7 +83,7 @@ describe("feature tools in the allow-list", () => { test("each placeholder answers that it isn't implemented, as a failure", async () => { for (const id of FEATURE_TOOLS) { - // Tested in its own file once implemented. + // Implemented tools have their own test file. if (id === "dbt_cte_profile") continue const tool = await initTool(EXPOSED[id] as any) expect(tool.id).toBe(id) diff --git a/packages/opencode/test/altimate/dbt-cte-profile.test.ts b/packages/opencode/test/altimate/dbt-cte-profile.test.ts index 28ce9eeef0..0a823b5b15 100644 --- a/packages/opencode/test/altimate/dbt-cte-profile.test.ts +++ b/packages/opencode/test/altimate/dbt-cte-profile.test.ts @@ -1,7 +1,7 @@ // dbt_cte_profile: the CTE list, and one timed count query per CTE. The dbt adapter is replaced through // `dbtAdapterInternals`, so no warehouse, Python or dbt is needed. import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, test } from "bun:test" -import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises" +import { mkdir, mkdtemp, rm, symlink, writeFile } from "node:fs/promises" import os from "node:os" import path from "node:path" import { Effect } from "effect" @@ -70,6 +70,24 @@ beforeAll(async () => { await writeFile(path.join(project, "models", "staging", "plain.sql"), "select 1 as id\n") await writeFile(path.join(project, "models", "marts", "dup.sql"), "with a as (select 1) select * from a") await writeFile(path.join(project, "models", "staging", "dup.sql"), "with b as (select 1) select * from b") + await writeFile( + path.join(project, "models", "marts", "twins.sql"), + '{% if is_incremental() %}\nwith "Foo" as (select 1 as a), foo as (select 2 as a), keep as (select 3 as a)\nselect * from keep\n{% endif %}\n', + ) + await writeFile( + path.join(project, "models", "marts", "second.sql"), + "with a as (select 1 as x) select * from a;\nwith b as (select 2 as x) select * from b\n", + ) + await mkdir(path.join(project, "target"), { recursive: true }) + await writeFile( + path.join(project, "target", "manifest.json"), + JSON.stringify({ + nodes: { + "model.jaffle_shop.customers": { resource_type: "model", name: "customers", original_file_path: "models/marts/customers.sql" }, + "model.jaffle_shop.gone": { resource_type: "model", name: "gone", original_file_path: "models/marts/gone.sql" }, + }, + }), + ) await writeFile(path.join(project, "dbt_packages", "pkg", "models", "ignored.sql"), "with z as (select 1) select * from z") }) @@ -97,9 +115,9 @@ describe("dbt_cte_profile list", () => { expect(result.metadata.success).toBe(true) expect(result.metadata.file).toBe(path.join("models", "marts", "customers.sql")) expect(result.metadata.ctes).toEqual([ - { name: "orders", line: 4, index: 0, clause: expect.any(Number) }, - { name: '"Big Spenders"', line: 8, index: 1, clause: expect.any(Number) }, - { name: "final", line: 12, index: 2, clause: expect.any(Number) }, + { name: "orders", line: 4, index: 0, clause: expect.any(Number), position: 0 }, + { name: '"Big Spenders"', line: 8, index: 1, clause: expect.any(Number), position: 1 }, + { name: "final", line: 12, index: 2, clause: expect.any(Number), position: 2 }, ]) expect(result.output).toContain("3 CTEs") expect(queries).toEqual([]) @@ -132,7 +150,146 @@ describe("dbt_cte_profile list", () => { }) }) +describe("dbt_cte_profile model lookup", () => { + test("finds a model by unique_id through the manifest, and takes an absolute path", async () => { + const tool = await initTool(DbtCteProfileTool) + const byId = await tool.execute({ model: "model.jaffle_shop.customers" }, ctx) + expect(byId.metadata.success).toBe(true) + const absolute = await tool.execute({ model: path.join(project, "models", "marts", "customers.sql") }, ctx) + expect(absolute.metadata.success).toBe(true) + expect(absolute.metadata.file).toBe(path.join("models", "marts", "customers.sql")) + }) + + test("a unique_id the manifest does not hold is refused, never read as a name", async () => { + const tool = await initTool(DbtCteProfileTool) + const other = await tool.execute({ model: "model.other_package.customers" }, ctx) + expect(other.metadata.success).toBe(false) + expect(other.output).toContain("is not in target/manifest.json") + // The manifest knows this model but its file is gone: the name search must not answer for another package's. + const gone = await tool.execute({ model: "model.jaffle_shop.gone" }, ctx) + expect(gone.metadata.success).toBe(false) + }) + + test("a folder, or a path that is not a .sql file, is a failure, not a crash", async () => { + const tool = await initTool(DbtCteProfileTool) + const folder = await tool.execute({ model: "models/marts/" }, ctx) + expect(folder.metadata.success).toBe(false) + expect(folder.output).toContain("not a .sql file") + const config = await tool.execute({ model: "models/../dbt_project.yml" }, ctx) + expect(config.metadata.success).toBe(false) + expect(config.output).toContain("not a .sql file") + const dirNamedSql = path.join(project, "models", "dir.sql") + await mkdir(dirNamedSql, { recursive: true }) + const named = await tool.execute({ model: "models/dir.sql" }, ctx) + expect(named.metadata.success).toBe(false) + await rm(dirNamedSql, { recursive: true }) + }) + + test("a symlink out of the project is refused", async () => { + const outside = await mkdtemp(path.join(os.tmpdir(), "dbt-cte-outside-")) + await writeFile(path.join(outside, "secret.sql"), "with s as (select 1) select * from s") + await symlink(path.join(outside, "secret.sql"), path.join(project, "models", "link.sql")) + const tool = await initTool(DbtCteProfileTool) + const result = await tool.execute({ model: "models/link.sql" }, ctx) + expect(result.metadata.success).toBe(false) + expect(result.output).toContain("outside the dbt project") + await rm(path.join(project, "models", "link.sql")) + await rm(outside, { recursive: true, force: true }) + }) + + test("the list is read from the session's project, not the one the adapter was set up for", async () => { + const other = await mkdtemp(path.join(os.tmpdir(), "dbt-cte-other-")) + await writeFile(path.join(other, "dbt_project.yml"), "name: other\n") + await mkdir(path.join(other, "models"), { recursive: true }) + await writeFile(path.join(other, "models", "customers.sql"), "with only_here as (select 1) select * from only_here") + const before = process.cwd() + process.chdir(other) + try { + const tool = await initTool(DbtCteProfileTool) + const result = await tool.execute({ model: "customers" }, ctx) + expect((result.metadata.ctes as { name: string }[]).map((c) => c.name)).toEqual(["only_here"]) + const run = await tool.execute({ model: "customers", cte: "only_here" }, ctx) + expect(run.metadata.success).toBe(false) + expect(run.metadata.refusal).toBe("other_project") + } finally { + process.chdir(before) + await rm(other, { recursive: true, force: true }) + } + }) + + test("an oversize model is refused", async () => { + await writeFile(path.join(project, "models", "huge.sql"), "-- " + "x".repeat(4 * 1024 * 1024 + 10)) + const tool = await initTool(DbtCteProfileTool) + const result = await tool.execute({ model: "huge" }, ctx) + expect(result.metadata.success).toBe(false) + expect(result.output).toContain("largest file read") + await rm(path.join(project, "models", "huge.sql")) + }) +}) + +describe("dbt_cte_profile same-named CTEs", () => { + test("two CTEs whose names differ only in case are listed apart, and each is reachable by its exact name", async () => { + const tool = await initTool(DbtCteProfileTool) + const list = await tool.execute({ model: "twins" }, ctx) + expect((list.metadata.ctes as { name: string; position: number }[]).map((c) => [c.name, c.position])).toEqual([['"Foo"', 0], ["foo", 1], ["keep", 2]]) + const exact = await tool.execute({ model: "twins", cte: "foo" }, ctx) + expect(exact.metadata.position).toBe(1) + expect(queries[0].sql).toEndWith("FROM foo") + const quoted = await tool.execute({ model: "twins", cte: '"Foo"' }, ctx) + expect(quoted.metadata.position).toBe(0) + expect(queries[1].sql).toEndWith('FROM "Foo"') + // Letter case alone does not pick between two matches. + const loose = await tool.execute({ model: "twins", cte: "FOO" }, ctx) + expect(loose.metadata.success).toBe(false) + expect(loose.output).toContain("positions 0, 1") + expect(queries).toHaveLength(2) + }) + + test("position picks one CTE, and a name that disagrees with it is refused", async () => { + const tool = await initTool(DbtCteProfileTool) + const result = await tool.execute({ model: "twins", position: 2 }, ctx) + expect(result.metadata.cte).toBe("keep") + const wrong = await tool.execute({ model: "twins", cte: "foo", position: 2 }, ctx) + expect(wrong.metadata.success).toBe(false) + const missing = await tool.execute({ model: "twins", position: 9 }, ctx) + expect(missing.metadata.success).toBe(false) + expect(queries).toHaveLength(1) + }) + + test("a CTE of a second WITH clause has no query of its own and runs nothing", async () => { + const tool = await initTool(DbtCteProfileTool) + const first = await tool.execute({ model: "second", cte: "a" }, ctx) + expect(first.metadata.success).toBe(true) + const later = await tool.execute({ model: "second", cte: "b" }, ctx) + expect(later.metadata.success).toBe(false) + expect(later.output).toContain("second WITH clause") + expect(queries).toHaveLength(1) + }) +}) + describe("dbt_cte_profile run", () => { + test("every failure carries an error, so the CLI and MCP report it as one", async () => { + const tool = await initTool(DbtCteProfileTool) + const results = [ + await tool.execute({ model: "nope" }, ctx), + await tool.execute({ model: "customers", cte: "nope" }, ctx), + await tool.execute({ model: "customers", cte: "orders" }, { ...ctx, extra: { forceReadOnly: true } }), + ] + for (const result of results) { + expect(result.metadata.success).toBe(false) + expect(typeof result.metadata.error).toBe("string") + } + }) + + test("a query that outlives the call's abort ends the wait at the abort", async () => { + respond = () => new Promise(() => {}) + const controller = new AbortController() + const tool = await initTool(DbtCteProfileTool) + const run = tool.execute({ model: "customers", cte: "orders" }, { ...ctx, abort: controller.signal }) + setTimeout(() => controller.abort(new Error("stopped")), 50) + await expect(run).rejects.toThrow("stopped") + }) + test("runs one count query over the CTE and the earlier ones, limit 1, and reads the count by position", async () => { const tool = await initTool(DbtCteProfileTool) const result = await tool.execute({ model: "customers", cte: "Big Spenders" }, ctx) From 541d1bee195d4b574d930e2aae74889fe176ab0d Mon Sep 17 00:00:00 2001 From: Harish Kumar Date: Fri, 13 Nov 2026 17:36:33 +0530 Subject: [PATCH 05/26] fix: dbt_cte_profile ignores Jinja blocks that hold SQL strings, keeps WITH RECURSIVE, reads $ and non-ASCII names, and refuses another package's model Co-Authored-By: Claude Sonnet 5.5 --- .../opencode/src/altimate/tools/cte-detect.ts | 49 +++++++++++++------ .../src/altimate/tools/dbt-cte-profile.ts | 16 ++++-- .../opencode/test/altimate/cte-detect.test.ts | 26 ++++++++-- .../test/altimate/dbt-cte-profile.test.ts | 5 ++ 4 files changed, 73 insertions(+), 23 deletions(-) diff --git a/packages/opencode/src/altimate/tools/cte-detect.ts b/packages/opencode/src/altimate/tools/cte-detect.ts index a4eea27eea..13205dff69 100644 --- a/packages/opencode/src/altimate/tools/cte-detect.ts +++ b/packages/opencode/src/altimate/tools/cte-detect.ts @@ -21,6 +21,8 @@ export interface CteInfo { index: number /** Offset of the WITH keyword that opens this CTE's clause. */ withClauseStart: number + /** The clause is `WITH RECURSIVE`. */ + recursive?: boolean } /** Position just past the string literal, quoted identifier or comment (`--`, block, Jinja) that starts at `pos`; `pos` itself when none does. An unterminated one runs to the end of the text. */ @@ -85,6 +87,23 @@ function findMatchingClosingParen(text: string, openParenPos: number): number { return -1 } +/** + * Position just past the Jinja tag at `pos`, and past its whole block when it opens one whose text + * is not the model's own SQL (`{% set x %}…{% endset %}`, `{% call %}`, `{% macro %}`): a query + * written there is a string for dbt to run, not a CTE of the model. + */ +function skipJinjaBlock(text: string, pos: number): number { + const close = text.indexOf("%}", pos + 2) + if (close === -1) return text.length + const tag = /^\{%-?\s*(set|call|macro)\b([^%]*)/.exec(text.substring(pos, close + 2)) + // `{% set x = … %}` is an inline set, with no block to skip. + if (!tag || (tag[1] === "set" && tag[2].includes("="))) return close + 2 + const end = new RegExp(`\\{%-?\\s*end${tag[1]}\\b[^%]*%\\}`, "g") + end.lastIndex = close + 2 + const found = end.exec(text) + return found ? found.index + found[0].length : text.length +} + /** Every WITH keyword outside comments, strings and parentheses: a WITH inside a CTE body or a subquery belongs to that query, not to the model. */ function findTopLevelWithKeywords(text: string): number[] { const positions: number[] = [] @@ -98,6 +117,10 @@ function findTopLevelWithKeywords(text: string): number[] { continue } const char = text[pos] + if (char === "{" && text[pos + 1] === "%") { + pos = skipJinjaBlock(text, pos) + continue + } if (char === "(") depth++ else if (char === ")") depth = Math.max(0, depth - 1) else if (depth === 0 && (char === "w" || char === "W") && !/[A-Za-z0-9_$]/.test(text[pos - 1] ?? " ")) { @@ -123,11 +146,12 @@ function readIdentifier(text: string, pos: number): number { if (close === end || text[close - 1] !== char || close - end < 2) return -1 end = close } else if (char === "[") { - const close = text.indexOf("]", end + 1) - if (close === -1) return -1 - end = close + 1 + const bracketed = /\[[^\]\n]{1,256}\]/y + bracketed.lastIndex = end + if (!bracketed.test(text)) return -1 + end = bracketed.lastIndex } else { - const bare = /[A-Za-z_][A-Za-z0-9_]*/y + const bare = /[\p{L}_][\p{L}\p{N}_$]*/uy bare.lastIndex = end if (!bare.test(text)) return -1 end = bare.lastIndex @@ -140,9 +164,10 @@ function readIdentifier(text: string, pos: number): number { /** The CTEs of the WITH clause that opens at `withPos`: each `name [(columns)] AS (body)`, separated by commas, up to the first thing that is not one. */ function parseWithClause(text: string, withPos: number, ctes: CteInfo[]): void { let pos = skipTrivia(text, withPos + 4, true) - const recursive = /recursive\b/iy - recursive.lastIndex = pos - if (recursive.test(text)) pos = skipTrivia(text, pos + "recursive".length, true) + const keyword = /recursive\b/iy + keyword.lastIndex = pos + const recursive = keyword.test(text) + if (recursive) pos = skipTrivia(text, pos + "recursive".length, true) const asOpen = /as(?:\s+(?:not\s+)?materialized)?\s*\(/iy for (let index = 0; ; index++) { @@ -174,6 +199,7 @@ function parseWithClause(text: string, withPos: number, ctes: CteInfo[]): void { queryEnd: close, index, withClauseStart: withPos, + ...(recursive && { recursive }), }) pos = skipTrivia(text, close + 1, true) if (text[pos] !== ",") return @@ -222,7 +248,7 @@ export function buildCountQuery(text: string, ctes: CteInfo[], target: CteInfo): const preamble = text.substring(0, target.withClauseStart).trim() let query = "" if (preamble) query += preamble + "\n\n" - query += "WITH " + definitions.join(",\n") + query += (target.recursive ? "WITH RECURSIVE " : "WITH ") + definitions.join(",\n") query += `\nSELECT COUNT(*) AS _profile_count FROM ${quoteSqlIdentifier(stripColumnList(target.name))}` return query } @@ -236,10 +262,3 @@ export function extractRowCount(data: Record[] | undefined): nu const first = Object.values(data[0])[0] return typeof first === "number" ? first : Number(first) || 0 } - -/** Zero-based line number of an offset. */ -export function lineOf(text: string, offset: number): number { - let line = 0 - for (let i = 0; i < offset && i < text.length; i++) if (text[i] === "\n") line++ - return line -} diff --git a/packages/opencode/src/altimate/tools/dbt-cte-profile.ts b/packages/opencode/src/altimate/tools/dbt-cte-profile.ts index 09c0ae0f1f..a6d5aae91e 100644 --- a/packages/opencode/src/altimate/tools/dbt-cte-profile.ts +++ b/packages/opencode/src/altimate/tools/dbt-cte-profile.ts @@ -6,7 +6,7 @@ import { loadRawManifest, findModel } from "../native/dbt/helpers" import { projectRoot } from "../native/connections/registry" import { executeQuery } from "../native/query-execution" import { dbtAdapterFor } from "./dbt-adapter-access" -import { buildCountQuery, detectCtes, extractRowCount, lineOf, stripColumnList, type CteInfo } from "./cte-detect" +import { buildCountQuery, detectCtes, extractRowCount, stripColumnList, type CteInfo } from "./cte-detect" /** Folders a model search never enters. */ const SKIPPED_DIRS = new Set(["target", "dbt_packages", "dbt_modules", "node_modules", "logs", "venv", "site-packages"]) @@ -28,7 +28,7 @@ function bareName(name: string): string { function inside(root: string, file: string): boolean { const rel = path.relative(root, file) - return rel !== "" && !rel.startsWith("..") && !path.isAbsolute(rel) + return rel !== "" && rel !== ".." && !rel.startsWith(".." + path.sep) && !path.isAbsolute(rel) } function realpath(p: string): string { @@ -76,6 +76,9 @@ function modelFile(root: string, model: string): { file?: string; error?: string const manifest = loadRawManifest(path.join(realRoot, "target", "manifest.json")) const node = manifest?.nodes ? findModel(manifest.nodes, model) : null if (node?.original_file_path) { + // dbt records the path relative to the model's own package: only the project's models can be read here. + const own = /^name:\s*['"]?([\w-]+)/m.exec(fs.readFileSync(path.join(realRoot, "dbt_project.yml"), "utf-8"))?.[1] + if (node.package_name && own && node.package_name !== own) return { error: `${model} belongs to the package ${node.package_name}: pass the path of its .sql file` } const candidate = path.resolve(realRoot, node.original_file_path) if (inside(realRoot, realpath(candidate)) && fs.existsSync(candidate)) return { file: realpath(candidate) } } @@ -131,9 +134,16 @@ export const DbtCteProfileTool = Tool.define("dbt_cte_profile", { const file = path.relative(realpath(root), located.file) if (args.cte === undefined && args.position === undefined) { + // The CTEs come in source order: count the lines once, not once per CTE. + let line = 1 + let counted = 0 + const lineAt = (offset: number) => { + for (; counted < offset; counted++) if (text[counted] === "\n") line++ + return line + } const list = ctes.map((c, position) => ({ name: stripColumnList(c.name), - line: lineOf(text, c.nameStart) + 1, + line: lineAt(c.nameStart), index: c.index, clause: c.withClauseStart, position, diff --git a/packages/opencode/test/altimate/cte-detect.test.ts b/packages/opencode/test/altimate/cte-detect.test.ts index 9bce1298b1..3d892e30ea 100644 --- a/packages/opencode/test/altimate/cte-detect.test.ts +++ b/packages/opencode/test/altimate/cte-detect.test.ts @@ -3,7 +3,6 @@ import { buildCountQuery, detectCtes as detectCtesInText, extractRowCount, - lineOf, quoteSqlIdentifier, stripColumnList, type CteInfo, @@ -1429,10 +1428,27 @@ select * from last` expect(performance.now() - started).toBeLessThan(1000) }) - it("reports zero-based lines", () => { - const sql = "with a as (select 1),\n b as (select 2)\nselect * from b" - const ctes = detectCtesInText(sql) - expect(ctes.map((c) => lineOf(sql, c.nameStart))).toEqual([0, 1]) + it("keeps WITH RECURSIVE in the count query, and a WITH written inside a Jinja block is not the model's", () => { + const recursive = "with recursive r as (select 1 as n union all select n + 1 from r where n < 3) select * from r" + const ctes = detectCtesInText(recursive) + expect(buildCountQuery(recursive, ctes, ctes[0])).toStartWith("WITH RECURSIVE r as (") + const block = "{% set q %}with z as (select 1) select * from z{% endset %}\nwith a as (select 1), b as (select 2) select * from b" + const found = detectCtesInText(block) + expect(found.map((c) => c.name)).toEqual(["a", "b"]) + expect(buildCountQuery(block, found, found[1])).toContain("b as (\nselect 2\n)") + const inline = "{% set q = 'with z as (select 1)' %}\nwith a as (select 1) select * from a" + expect(detectCtesInText(inline).map((c) => c.name)).toEqual(["a"]) + const macro = "{% macro m() %}with z as (select 1) select * from z{% endmacro %}\nwith a as (select 1) select * from a" + expect(detectCtesInText(macro).map((c) => c.name)).toEqual(["a"]) + }) + + it("reads $ and non-ASCII letters in a name, and survives crafted input in linear time", () => { + expect(detectCtesInText("with my$cte as (select 1), é_t as (select 2) select * from é_t").map((c) => c.name)).toEqual(["my$cte", "é_t"]) + for (const unit of ["with [", "with {%", "with ((", "with a as (", "with '", "with /*", "with \"x"]) { + const started = performance.now() + detectCtesInText(unit.repeat(200_000)) + expect(performance.now() - started).toBeLessThan(2000) + } }) }) diff --git a/packages/opencode/test/altimate/dbt-cte-profile.test.ts b/packages/opencode/test/altimate/dbt-cte-profile.test.ts index 0a823b5b15..9c6dcbe6be 100644 --- a/packages/opencode/test/altimate/dbt-cte-profile.test.ts +++ b/packages/opencode/test/altimate/dbt-cte-profile.test.ts @@ -85,6 +85,7 @@ beforeAll(async () => { nodes: { "model.jaffle_shop.customers": { resource_type: "model", name: "customers", original_file_path: "models/marts/customers.sql" }, "model.jaffle_shop.gone": { resource_type: "model", name: "gone", original_file_path: "models/marts/gone.sql" }, + "model.pkg.customers": { resource_type: "model", name: "customers", package_name: "pkg", original_file_path: "models/marts/customers.sql" }, }, }), ) @@ -168,6 +169,10 @@ describe("dbt_cte_profile model lookup", () => { // The manifest knows this model but its file is gone: the name search must not answer for another package's. const gone = await tool.execute({ model: "model.jaffle_shop.gone" }, ctx) expect(gone.metadata.success).toBe(false) + // A package's model is recorded relative to its own package: never read as the project's file of that path. + const packaged = await tool.execute({ model: "model.pkg.customers" }, ctx) + expect(packaged.metadata.success).toBe(false) + expect(packaged.output).toContain("belongs to the package pkg") }) test("a folder, or a path that is not a .sql file, is a failure, not a crash", async () => { From 85abfe2ccbf95b1c94560cafc00e428f20ac06a9 Mon Sep 17 00:00:00 2001 From: Harish Kumar Date: Fri, 13 Nov 2026 21:56:23 +0530 Subject: [PATCH 06/26] fix: dbt_cte_profile builds its count query from the model's own text, refuses a CTE inside an open Jinja block, and reports a timeout as one The count query is the model's text up to the end of the chosen CTE, so Jinja between CTEs is kept; when that cut leaves a Jinja block open the tool says so instead of sending a query dbt cannot compile. Comments between AS, NOT and MATERIALIZED, nested set blocks, dollar-quoted strings, quoted ")" in column lists, and names with $ or non-ASCII letters are read correctly. A model name shared by several manifest entries is refused rather than guessed. Co-Authored-By: Claude Sonnet 5.5 --- .../opencode/src/altimate/tools/cte-detect.ts | 162 ++++++++++++++---- .../src/altimate/tools/dbt-cte-profile.ts | 34 +++- .../opencode/test/altimate/cte-detect.test.ts | 103 +++++++++-- .../test/altimate/dbt-cte-profile.test.ts | 36 +++- 4 files changed, 283 insertions(+), 52 deletions(-) diff --git a/packages/opencode/src/altimate/tools/cte-detect.ts b/packages/opencode/src/altimate/tools/cte-detect.ts index 13205dff69..d4fd36a7c4 100644 --- a/packages/opencode/src/altimate/tools/cte-detect.ts +++ b/packages/opencode/src/altimate/tools/cte-detect.ts @@ -40,6 +40,14 @@ function skipLiteral(text: string, pos: number): number { } return text.length } + if (char === "$" && !/[\p{L}\p{N}_$]/u.test(text[pos - 1] ?? " ")) { + // A dollar-quoted string (`$$ … $$`, `$tag$ … $tag$`) runs to its matching tag. + const tag = /^\$(?:[A-Za-z_][A-Za-z0-9_]*)?\$/.exec(text.substring(pos, pos + 70))?.[0] + if (tag) { + const end = text.indexOf(tag, pos + tag.length) + return end === -1 ? text.length : end + tag.length + } + } const close = char === "-" && next === "-" ? "\n" : char === "/" && next === "*" ? "*/" : char === "{" && next === "#" ? "#}" : null if (close === null) return pos const end = text.indexOf(close, pos + 2) @@ -87,6 +95,67 @@ function findMatchingClosingParen(text: string, openParenPos: number): number { return -1 } +interface JinjaTag { + /** Tag name (`if`, `set`, `macro`…), without the `end` prefix. */ + name: string + /** The tag closes a block (`{% endif %}`). */ + closes: boolean + /** The tag opens a block: not an `end` tag, `else`/`elif`, or an inline `{% set x = … %}`. */ + opens: boolean + /** Offset just past the tag. */ + end: number +} + +const BLOCK_TAGS = new Set(["if", "for", "macro", "call", "filter", "with", "set", "raw"]) + +/** The block tag of the `{% … %}` that spans `at` to `close` (the offset of its `%}`); undefined for any other tag. */ +function parseTag(text: string, at: number, close: number): JinjaTag | undefined { + const head = /^\{%-?\s*(end)?(\w+)/.exec(text.substring(at, Math.min(close + 2, at + 200))) + if (!head || !BLOCK_TAGS.has(head[2])) return undefined + const closes = head[1] !== undefined + // `{% set x = … %}` is an inline set; `{% set x %}` and `{% set x | filter %}` open a block. + const inline = head[2] === "set" && /^\{%-?\s*set\s+[\w.,\s]+=(?!=)/.test(text.substring(at, close + 2)) + return { name: head[2], closes, opens: !closes && !inline, end: close + 2 } +} + +/** + * The Jinja statement tags of `text` from `from`, in order, in one linear pass. Comments (`{# #}`) and the + * inside of `{% raw %}` are not tags; an unterminated comment or tag ends the text. + */ +function* jinjaTags(text: string, from: number): Generator { + let pos = from + for (;;) { + const at = text.indexOf("{", pos) + if (at === -1 || at + 1 >= text.length) return + const kind = text[at + 1] + if (kind !== "#" && kind !== "%") { + pos = at + 1 + continue + } + const close = text.indexOf(kind === "#" ? "#}" : "%}", at + 2) + if (close === -1) return + pos = close + 2 + if (kind === "#") continue + const tag = parseTag(text, at, close) + if (!tag) continue + if (tag.name === "raw" && tag.opens) { + // Nothing inside `{% raw %}` is Jinja: the block runs to its end tag. + const end = /\{%-?\s*endraw\b/g + end.lastIndex = pos + const found = end.exec(text) + if (!found) { + yield { name: "raw", closes: false, opens: true, end: text.length } + return + } + const endClose = text.indexOf("%}", found.index) + pos = endClose === -1 ? text.length : endClose + 2 + yield { name: "raw", closes: false, opens: false, end: pos } + continue + } + yield tag + } +} + /** * Position just past the Jinja tag at `pos`, and past its whole block when it opens one whose text * is not the model's own SQL (`{% set x %}…{% endset %}`, `{% call %}`, `{% macro %}`): a query @@ -95,13 +164,17 @@ function findMatchingClosingParen(text: string, openParenPos: number): number { function skipJinjaBlock(text: string, pos: number): number { const close = text.indexOf("%}", pos + 2) if (close === -1) return text.length - const tag = /^\{%-?\s*(set|call|macro)\b([^%]*)/.exec(text.substring(pos, close + 2)) - // `{% set x = … %}` is an inline set, with no block to skip. - if (!tag || (tag[1] === "set" && tag[2].includes("="))) return close + 2 - const end = new RegExp(`\\{%-?\\s*end${tag[1]}\\b[^%]*%\\}`, "g") - end.lastIndex = close + 2 - const found = end.exec(text) - return found ? found.index + found[0].length : text.length + const first = parseTag(text, pos, close) + if (!first || !first.opens || (first.name !== "set" && first.name !== "call" && first.name !== "macro")) return close + 2 + // Blocks of the same kind nest: the block ends at the end tag that matches this one. + let depth = 1 + for (const tag of jinjaTags(text, close + 2)) { + if (tag.name !== first.name) continue + if (tag.closes) depth-- + else if (tag.opens) depth++ + if (depth === 0) return tag.end + } + return text.length } /** Every WITH keyword outside comments, strings and parentheses: a WITH inside a CTE body or a subquery belongs to that query, not to the model. */ @@ -161,6 +234,23 @@ function readIdentifier(text: string, pos: number): number { } } +/** Offset of the `(` that opens a CTE body after `AS [NOT] [MATERIALIZED]` starting at `pos`; -1 when the text there is not such a header. */ +function skipHeader(text: string, pos: number): number { + let at = pos + const word = (w: string) => { + const re = new RegExp(`${w}\\b`, "iy") + re.lastIndex = at + if (!re.test(text)) return false + at = skipTrivia(text, at + w.length, true) + return true + } + if (!word("as")) return -1 + if (word("not")) { + if (!word("materialized")) return -1 + } else word("materialized") + return text[at] === "(" ? at : -1 +} + /** The CTEs of the WITH clause that opens at `withPos`: each `name [(columns)] AS (body)`, separated by commas, up to the first thing that is not one. */ function parseWithClause(text: string, withPos: number, ctes: CteInfo[]): void { let pos = skipTrivia(text, withPos + 4, true) @@ -169,7 +259,6 @@ function parseWithClause(text: string, withPos: number, ctes: CteInfo[]): void { const recursive = keyword.test(text) if (recursive) pos = skipTrivia(text, pos + "recursive".length, true) - const asOpen = /as(?:\s+(?:not\s+)?materialized)?\s*\(/iy for (let index = 0; ; index++) { const nameStart = pos const identifierEnd = readIdentifier(text, nameStart) @@ -185,10 +274,9 @@ function parseWithClause(text: string, withPos: number, ctes: CteInfo[]): void { nameEnd = close + 1 pos = skipTrivia(text, nameEnd, false) } - asOpen.lastIndex = pos - const header = asOpen.exec(text) - if (!header) return - const open = pos + header[0].length - 1 + // `AS [NOT] [MATERIALIZED] (`, with comments or Jinja tags allowed between the words. + const open = skipHeader(text, pos) + if (open === -1) return const close = findMatchingClosingParen(text, open) if (close === -1) return ctes.push({ @@ -216,41 +304,47 @@ export function detectCtes(text: string): CteInfo[] { /** Drop a trailing column list: `my_cte (id, name)` -> `my_cte`. */ export function stripColumnList(rawName: string): string { - return rawName.replace(/\s*\([^)]*\)\s*$/, "").trim() + const end = readIdentifier(rawName, 0) + return end === -1 || !rawName.substring(end).trimStart().startsWith("(") ? rawName.trim() : rawName.substring(0, end) } export function quoteSqlIdentifier(identifier: string): string { if (identifier.match(/^["'`[]/) || identifier.includes(".")) return identifier - if (!identifier.match(/^[a-zA-Z_][a-zA-Z0-9_]*$/)) return `"${identifier}"` + if (!identifier.match(/^[\p{L}_][\p{L}\p{N}_$]*$/u)) return `"${identifier}"` return identifier } +/** Name of the first Jinja block (`if`, `for`, `macro`, `call`, `filter`, `with`, `raw`, or `set` as a block) opened in `text` and never closed there; undefined when every block is closed. Tags inside Jinja comments and `{% raw %}` do not count. */ +export function unclosedJinjaBlock(text: string): string | undefined { + const open: string[] = [] + const counts = new Map() + for (const tag of jinjaTags(text, 0)) { + if (tag.closes) { + // An end tag closes the nearest open block of its kind, and any block opened inside it. + if (!counts.get(tag.name)) continue + for (let top = open.pop(); top !== undefined; top = open.pop()) { + counts.set(top, counts.get(top)! - 1) + if (top === tag.name) break + } + } else if (tag.opens) { + open.push(tag.name) + counts.set(tag.name, (counts.get(tag.name) ?? 0) + 1) + } + } + return open[0] +} + /** - * `SELECT COUNT(*)` over one CTE, keeping the preamble (dbt configs, variables) - * and every earlier CTE of the same WITH clause, so the query compiles on its own. + * `SELECT COUNT(*)` over one CTE: the model's text up to the end of that CTE (dbt configs, variables, + * Jinja between CTEs and every earlier CTE of the WITH clause, as written), so the query compiles on its own. * Undefined for a CTE of a second WITH clause in the same file, which has no standalone query. */ export function buildCountQuery(text: string, ctes: CteInfo[], target: CteInfo): string | undefined { // The text before a later WITH holds the earlier clause's whole query: nothing valid stands alone there. if (ctes.some((c) => c.withClauseStart < target.withClauseStart)) return undefined - const upToTarget = ctes.filter((c) => c.withClauseStart === target.withClauseStart && c.index <= target.index) - - const definitions: string[] = [] - for (const cte of upToTarget) { - if (cte.queryStart <= cte.nameStart) continue - // Slice from the name to the body so column lists and comments before AS are kept. - const header = text.substring(cte.nameStart, cte.queryStart) - const body = text.substring(cte.queryStart, cte.queryEnd) - definitions.push(`${header}\n${body}\n)`) - } - if (definitions.length === 0) return undefined - - const preamble = text.substring(0, target.withClauseStart).trim() - let query = "" - if (preamble) query += preamble + "\n\n" - query += (target.recursive ? "WITH RECURSIVE " : "WITH ") + definitions.join(",\n") - query += `\nSELECT COUNT(*) AS _profile_count FROM ${quoteSqlIdentifier(stripColumnList(target.name))}` - return query + if (target.queryStart <= target.nameStart) return undefined + const head = text.substring(0, target.queryEnd + 1).trim() + return `${head}\nSELECT COUNT(*) AS _profile_count FROM ${quoteSqlIdentifier(stripColumnList(target.name))}` } /** diff --git a/packages/opencode/src/altimate/tools/dbt-cte-profile.ts b/packages/opencode/src/altimate/tools/dbt-cte-profile.ts index a6d5aae91e..ade5450efa 100644 --- a/packages/opencode/src/altimate/tools/dbt-cte-profile.ts +++ b/packages/opencode/src/altimate/tools/dbt-cte-profile.ts @@ -4,9 +4,10 @@ import z from "zod" import { Tool } from "../../tool/tool" import { loadRawManifest, findModel } from "../native/dbt/helpers" import { projectRoot } from "../native/connections/registry" +import { isQueryTimeoutError } from "@altimateai/drivers/types" import { executeQuery } from "../native/query-execution" import { dbtAdapterFor } from "./dbt-adapter-access" -import { buildCountQuery, detectCtes, extractRowCount, stripColumnList, type CteInfo } from "./cte-detect" +import { buildCountQuery, detectCtes, extractRowCount, stripColumnList, unclosedJinjaBlock, type CteInfo } from "./cte-detect" /** Folders a model search never enters. */ const SKIPPED_DIRS = new Set(["target", "dbt_packages", "dbt_modules", "node_modules", "logs", "venv", "site-packages"]) @@ -74,10 +75,18 @@ function modelFile(root: string, model: string): { file?: string; error?: string } try { const manifest = loadRawManifest(path.join(realRoot, "target", "manifest.json")) - const node = manifest?.nodes ? findModel(manifest.nodes, model) : null + const own = /^name:\s*['"]?([\w-]+)/m.exec(fs.readFileSync(path.join(realRoot, "dbt_project.yml"), "utf-8"))?.[1] + let node = manifest?.nodes ? findModel(manifest.nodes, model) : null + if (node && !model.startsWith("model.")) { + // The project's own model of that name comes before a package's; versions of one model share a name and the manifest can't say which is meant. + const named = Object.values(manifest.nodes).filter((n) => n.resource_type === "model" && n.name === model) + const mine = named.filter((n) => !own || !n.package_name || n.package_name === own) + const pool = mine.length > 0 ? mine : named + if (pool.length > 1) return { error: `${model} matches ${pool.length} models (${pool.map((n) => n.unique_id).join(", ")}): pass the path of the one to profile` } + node = pool[0] + } if (node?.original_file_path) { // dbt records the path relative to the model's own package: only the project's models can be read here. - const own = /^name:\s*['"]?([\w-]+)/m.exec(fs.readFileSync(path.join(realRoot, "dbt_project.yml"), "utf-8"))?.[1] if (node.package_name && own && node.package_name !== own) return { error: `${model} belongs to the package ${node.package_name}: pass the path of its .sql file` } const candidate = path.resolve(realRoot, node.original_file_path) if (inside(realRoot, realpath(candidate)) && fs.existsSync(candidate)) return { file: realpath(candidate) } @@ -180,7 +189,18 @@ export const DbtCteProfileTool = Tool.define("dbt_cte_profile", { } const query = buildCountQuery(text, ctes, target) const name = stripColumnList(target.name) - if (!query) return failure("no query", `${name} is in a second WITH clause of ${args.model}, which has no query of its own to count.`, { model: args.model, cte: name }) + if (!query) { + const block = unclosedJinjaBlock(text.substring(0, target.withClauseStart)) + return failure( + "no query", + block + ? `${name} sits in a WITH clause of ${args.model} that is one branch of a Jinja {% ${block} %} block, so it has no query of its own to count.` + : `${name} is in a second WITH clause of ${args.model}, which has no query of its own to count.`, + { model: args.model, cte: name }, + ) + } + const block = unclosedJinjaBlock(query) + if (block) return failure("no query", `${name} is inside a Jinja {% ${block} %} block of ${args.model}, which cannot be cut off at that CTE and stay valid: compile the model and profile the compiled SQL, or move the WITH out of the block.`, { model: args.model, cte: name }) const access = await dbtAdapterFor(ctx) if (!access.adapter) return failure("not run", access.reason ?? "the dbt adapter is not available", { model: args.model, cte: name, refusal: access.refusal }) @@ -189,7 +209,7 @@ export const DbtCteProfileTool = Tool.define("dbt_cte_profile", { const started = performance.now() try { // The adapter has no cancel of its own: the run's timeout, budget and abort end the wait, not the query. - const result: { data?: Record[] } = await executeQuery({ type: "dbt" } as any, {}, { signal: ctx.abort, sessionID: ctx.sessionID }, () => + const result: { data?: Record[] } = await executeQuery({ type: "dbt" }, {}, { signal: ctx.abort, sessionID: ctx.sessionID }, () => adapter.immediatelyExecuteSQLWithLimit(query, `cte_profiler_${name}`, 1), ) const queryMs = Math.round(performance.now() - started) @@ -202,7 +222,9 @@ export const DbtCteProfileTool = Tool.define("dbt_cte_profile", { } catch (e) { if (ctx.abort.aborted) throw e const message = e instanceof Error ? e.message : String(e) - return failure("query failed", `The count query for ${name} failed: ${message}`, { model: args.model, cte: name }) + // A timeout gives up the wait, not the query: it may still be running on the warehouse. + const timedOut = isQueryTimeoutError(e) + return failure(timedOut ? "timed out" : "query failed", `The count query for ${name} ${timedOut ? "timed out and may still be running on the warehouse" : "failed"}: ${message}`, { model: args.model, cte: name, ...(timedOut && { timedOut }) }) } }, }) diff --git a/packages/opencode/test/altimate/cte-detect.test.ts b/packages/opencode/test/altimate/cte-detect.test.ts index 3d892e30ea..b50e6ccfdc 100644 --- a/packages/opencode/test/altimate/cte-detect.test.ts +++ b/packages/opencode/test/altimate/cte-detect.test.ts @@ -5,6 +5,7 @@ import { extractRowCount, quoteSqlIdentifier, stripColumnList, + unclosedJinjaBlock, type CteInfo, } from "../../src/altimate/tools/cte-detect" @@ -318,9 +319,8 @@ select * from source` const ctes = detectCtes(document) - // With unterminated comment, the parser may not detect the CTE correctly - // This test verifies graceful handling of malformed comments - expect(ctes.length).toBeGreaterThanOrEqual(0) + // The comment swallows the rest of the text, so there is no CTE to find. + expect(ctes).toEqual([]) }) it("should handle sequential line and block comments between name and AS", () => { @@ -1274,9 +1274,8 @@ select * from renamed` const ctes = detectCtes(document) - // The regex actually detects "comment" as a CTE name due to the unterminated comment - // This is expected behavior - the regex tries to parse what it can - expect(ctes.length).toBeGreaterThanOrEqual(0) + // The comment swallows the rest of the text, so there is no CTE to find. + expect(ctes).toEqual([]) }) }) @@ -1431,11 +1430,11 @@ select * from last` it("keeps WITH RECURSIVE in the count query, and a WITH written inside a Jinja block is not the model's", () => { const recursive = "with recursive r as (select 1 as n union all select n + 1 from r where n < 3) select * from r" const ctes = detectCtesInText(recursive) - expect(buildCountQuery(recursive, ctes, ctes[0])).toStartWith("WITH RECURSIVE r as (") + expect(buildCountQuery(recursive, ctes, ctes[0])).toStartWith("with recursive r as (") const block = "{% set q %}with z as (select 1) select * from z{% endset %}\nwith a as (select 1), b as (select 2) select * from b" const found = detectCtesInText(block) expect(found.map((c) => c.name)).toEqual(["a", "b"]) - expect(buildCountQuery(block, found, found[1])).toContain("b as (\nselect 2\n)") + expect(buildCountQuery(block, found, found[1])).toContain("b as (select 2)") const inline = "{% set q = 'with z as (select 1)' %}\nwith a as (select 1) select * from a" expect(detectCtesInText(inline).map((c) => c.name)).toEqual(["a"]) const macro = "{% macro m() %}with z as (select 1) select * from z{% endmacro %}\nwith a as (select 1) select * from a" @@ -1467,7 +1466,7 @@ describe("buildCountQuery", () => { expect(ctes.map((c) => c.name)).toEqual(["orders", "customers", "joined"]) const query = buildCountQuery(sql, ctes, ctes[1])! expect(query.startsWith("{{ config(materialized='table') }}")).toBe(true) - expect(query).toContain("WITH orders as (") + expect(query).toContain("with orders as (") expect(query).toContain("customers as (") expect(query).not.toContain("joined") expect(query.endsWith("SELECT COUNT(*) AS _profile_count FROM customers")).toBe(true) @@ -1485,14 +1484,14 @@ describe("buildCountQuery", () => { const text = "with a (x, y) as (select 1, 2) select * from a" const ctes = detectCtesInText(text) const query = buildCountQuery(text, ctes, ctes[0])! - expect(query).toContain("WITH a (x, y) as (") + expect(query).toContain("with a (x, y) as (") expect(query.endsWith("FROM a")).toBe(true) }) it("builds no query for a CTE of a second WITH clause, whose preamble would hold the first clause's query", () => { const text = "with a as (select 1) select * from a;\nwith b as (select 2) select * from b" const ctes = detectCtesInText(text) - expect(buildCountQuery(text, ctes, ctes.find((c) => c.name === "a")!)).toContain("WITH a as (") + expect(buildCountQuery(text, ctes, ctes.find((c) => c.name === "a")!)).toContain("with a as (") expect(buildCountQuery(text, ctes, ctes.find((c) => c.name === "b")!)).toBeUndefined() }) @@ -1511,6 +1510,88 @@ describe("buildCountQuery", () => { }) }) +describe("count query edge cases", () => { + it("keeps Jinja written between CTEs, and reports a block left open by the cut", () => { + const kept = "{% set n = 2 %}\nwith a as (select {{ n }}),\n{% if true %}b as (select 2),{% endif %}\nc as (select 3) select * from c" + const ctes = detectCtesInText(kept) + expect(buildCountQuery(kept, ctes, ctes[2])).toContain("{% if true %}b as (select 2),{% endif %}") + expect(unclosedJinjaBlock(buildCountQuery(kept, ctes, ctes[2])!)).toBeUndefined() + const open = "{% if x %}\nwith a as (select 1) select * from a\n{% endif %}" + const found = detectCtesInText(open) + expect(unclosedJinjaBlock(buildCountQuery(open, found, found[0])!)).toBe("if") + expect(unclosedJinjaBlock("{% set q %}x")).toBe("set") + expect(unclosedJinjaBlock("{% raw %}{% if x %}{% endraw %}")).toBeUndefined() + }) + + it("matches nested set blocks to their own end tag", () => { + const sql = "{% set outer %}{% set inner %}with fake as (select 1){% endset %}x{% endset %}\nwith real as (select 2) select * from real" + expect(detectCtesInText(sql).map((c) => c.name)).toEqual(["real"]) + }) + + it("reads a comment or Jinja tag between AS, NOT and MATERIALIZED and the body", () => { + expect(detectCtesInText("with a as /* c */ (select 1), b as not /* c */ materialized (select 2) select 1").map((c) => c.name)).toEqual(["a", "b"]) + expect(detectCtesInText("with a /* c */ as {# j #} materialized -- x\n(select 1) select 1").map((c) => c.name)).toEqual(["a"]) + }) + + it("strips a column list that holds a quoted ')' or a comment", () => { + expect(stripColumnList('a ("x)")')).toBe("a") + expect(stripColumnList("a (x /* ) */)")).toBe("a") + expect(stripColumnList("Big Spenders")).toBe("Big Spenders") + const text = 'with a ("x)", y) as (select 1, 2) select * from a' + const ctes = detectCtesInText(text) + expect(buildCountQuery(text, ctes, ctes[0])).toEndWith("FROM a") + }) + + it("a dollar-quoted string does not end the CTE body early", () => { + const text = "with a as (select $$ ) $$ as s), b as (select $tag$ ( $tag$ as t) select 1" + expect(detectCtesInText(text).map((c) => c.name)).toEqual(["a", "b"]) + }) + + it("tags inside a Jinja comment are not blocks, and a % inside a tag does not hide it", () => { + expect(unclosedJinjaBlock("{# {% if x %} #}with a as (select 1)")).toBeUndefined() + expect(unclosedJinjaBlock("{% if x %}{# {% endif %} #}with a as (select 1)")).toBe("if") + expect(unclosedJinjaBlock("{% if x % 2 == 0 %}with a as (select 1)")).toBe("if") + const sql = "{% set q %}{# {% endset %} #}with z as (select 1){% endset %}\nwith a as (select 2) select * from a" + expect(detectCtesInText(sql).map((c) => c.name)).toEqual(["a"]) + }) + + it("counts open Jinja blocks in linear time: 100k tags in a second", () => { + const sql = "{% if x %}".repeat(100_000) + "{% endfor %}".repeat(100_000) + const started = performance.now() + expect(unclosedJinjaBlock(sql)).toBe("if") + expect(performance.now() - started).toBeLessThan(1000) + }) + + it("unterminated tags, comments and many block tags are scanned in linear time", () => { + for (const unit of ["{% if x ", "{# c ", "{% set q %}", "{% set q | upper %}"]) { + const sql = unit.repeat(100_000) + "with a as (select 1) select 1" + const started = performance.now() + unclosedJinjaBlock(sql) + detectCtesInText(sql) + expect(performance.now() - started).toBeLessThan(1500) + } + }) + + it("a filtered set block is a block, and raw text holds no tags", () => { + expect(unclosedJinjaBlock("{% set q | upper %}x")).toBe("set") + expect(unclosedJinjaBlock("{% set q = 1 | upper %}x")).toBeUndefined() + const sql = "{% set q %}{% raw %}{% endset %}{% endraw %}with z as (select 1){% endset %}\nwith a as (select 2) select * from a" + expect(detectCtesInText(sql).map((c) => c.name)).toEqual(["a"]) + }) + + it("a dollar-quote tag may hold digits", () => { + expect(detectCtesInText("with a as (select $f1$ ) $f1$ as s), b as (select 2) select 1").map((c) => c.name)).toEqual(["a", "b"]) + }) + + it("writes a name with $ or a non-ASCII letter as it was declared", () => { + for (const name of ["my$cte", "Ünits"]) { + const text = `with ${name} as (select 1) select * from ${name}` + const ctes = detectCtesInText(text) + expect(buildCountQuery(text, ctes, ctes[0])).toEndWith(`FROM ${name}`) + } + }) +}) + describe("quoteSqlIdentifier", () => { it("leaves plain, quoted and dotted names alone and quotes the rest", () => { expect(quoteSqlIdentifier("my_cte")).toBe("my_cte") diff --git a/packages/opencode/test/altimate/dbt-cte-profile.test.ts b/packages/opencode/test/altimate/dbt-cte-profile.test.ts index 9c6dcbe6be..ecd93780d4 100644 --- a/packages/opencode/test/altimate/dbt-cte-profile.test.ts +++ b/packages/opencode/test/altimate/dbt-cte-profile.test.ts @@ -72,7 +72,11 @@ beforeAll(async () => { await writeFile(path.join(project, "models", "staging", "dup.sql"), "with b as (select 1) select * from b") await writeFile( path.join(project, "models", "marts", "twins.sql"), - '{% if is_incremental() %}\nwith "Foo" as (select 1 as a), foo as (select 2 as a), keep as (select 3 as a)\nselect * from keep\n{% endif %}\n', + 'with "Foo" as (select 1 as a), foo as (select 2 as a), keep as (select 3 as a)\nselect * from keep\n', + ) + await writeFile( + path.join(project, "models", "marts", "wrapped.sql"), + "{% if is_incremental() %}\nwith a as (select 1 as x) select * from a\n{% else %}\nwith b as (select 2 as x) select * from b\n{% endif %}\n", ) await writeFile( path.join(project, "models", "marts", "second.sql"), @@ -85,6 +89,8 @@ beforeAll(async () => { nodes: { "model.jaffle_shop.customers": { resource_type: "model", name: "customers", original_file_path: "models/marts/customers.sql" }, "model.jaffle_shop.gone": { resource_type: "model", name: "gone", original_file_path: "models/marts/gone.sql" }, + "model.jaffle_shop.dup.v1": { resource_type: "model", name: "sales", unique_id: "model.jaffle_shop.sales.v1", original_file_path: "models/marts/dup.sql" }, + "model.jaffle_shop.sales.v2": { resource_type: "model", name: "sales", unique_id: "model.jaffle_shop.sales.v2", original_file_path: "models/staging/dup.sql" }, "model.pkg.customers": { resource_type: "model", name: "customers", package_name: "pkg", original_file_path: "models/marts/customers.sql" }, }, }), @@ -295,6 +301,34 @@ describe("dbt_cte_profile run", () => { await expect(run).rejects.toThrow("stopped") }) + test("a name shared by several versions of one model is refused, not guessed", async () => { + const tool = await initTool(DbtCteProfileTool) + const result = await tool.execute({ model: "sales" }, ctx) + expect(result.metadata.success).toBe(false) + expect(result.output).toContain("matches 2 models") + }) + + test("a query that timed out is reported as timed out, since it may still be running", async () => { + respond = async () => { + throw Object.assign(new Error("exceeded 300000 ms"), { code: "QUERY_TIMEOUT" }) + } + const tool = await initTool(DbtCteProfileTool) + const result = await tool.execute({ model: "customers", cte: "orders" }, ctx) + expect(result.metadata.success).toBe(false) + expect(result.metadata.timedOut).toBe(true) + expect(result.output).toContain("may still be running") + }) + + test("a CTE inside a Jinja block that the query would cut open is refused, and nothing runs", async () => { + const tool = await initTool(DbtCteProfileTool) + for (const cte of ["a", "b"]) { + const result = await tool.execute({ model: "wrapped", cte }, ctx) + expect(result.metadata.success).toBe(false) + expect(result.output).toContain("Jinja") + } + expect(queries).toHaveLength(0) + }) + test("runs one count query over the CTE and the earlier ones, limit 1, and reads the count by position", async () => { const tool = await initTool(DbtCteProfileTool) const result = await tool.execute({ model: "customers", cte: "Big Spenders" }, ctx) From c04d485433e0c511f6bf426bf541c0f1a3c751fe Mon Sep 17 00:00:00 2001 From: Harish Kumar Date: Sat, 14 Nov 2026 08:00:15 +0530 Subject: [PATCH 07/26] fix: dbt_cte_profile refuses a CTE inside a Jinja block tag, and reads {%+ and tuple set tags Co-Authored-By: Claude Sonnet 5.5 --- packages/opencode/src/altimate/tools/cte-detect.ts | 8 ++++---- packages/opencode/test/altimate/cte-detect.test.ts | 3 +++ 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/packages/opencode/src/altimate/tools/cte-detect.ts b/packages/opencode/src/altimate/tools/cte-detect.ts index d4fd36a7c4..4935df22bb 100644 --- a/packages/opencode/src/altimate/tools/cte-detect.ts +++ b/packages/opencode/src/altimate/tools/cte-detect.ts @@ -106,15 +106,15 @@ interface JinjaTag { end: number } -const BLOCK_TAGS = new Set(["if", "for", "macro", "call", "filter", "with", "set", "raw"]) +const BLOCK_TAGS = new Set(["if", "for", "macro", "call", "filter", "with", "set", "raw", "block"]) /** The block tag of the `{% … %}` that spans `at` to `close` (the offset of its `%}`); undefined for any other tag. */ function parseTag(text: string, at: number, close: number): JinjaTag | undefined { - const head = /^\{%-?\s*(end)?(\w+)/.exec(text.substring(at, Math.min(close + 2, at + 200))) + const head = /^\{%[-+]?\s*(end)?(\w+)/.exec(text.substring(at, Math.min(close + 2, at + 200))) if (!head || !BLOCK_TAGS.has(head[2])) return undefined const closes = head[1] !== undefined // `{% set x = … %}` is an inline set; `{% set x %}` and `{% set x | filter %}` open a block. - const inline = head[2] === "set" && /^\{%-?\s*set\s+[\w.,\s]+=(?!=)/.test(text.substring(at, close + 2)) + const inline = head[2] === "set" && /^\{%[-+]?\s*set\s+[\w.,\s()]+=(?!=)/.test(text.substring(at, close + 2)) return { name: head[2], closes, opens: !closes && !inline, end: close + 2 } } @@ -140,7 +140,7 @@ function* jinjaTags(text: string, from: number): Generator { if (!tag) continue if (tag.name === "raw" && tag.opens) { // Nothing inside `{% raw %}` is Jinja: the block runs to its end tag. - const end = /\{%-?\s*endraw\b/g + const end = /\{%[-+]?\s*endraw\b/g end.lastIndex = pos const found = end.exec(text) if (!found) { diff --git a/packages/opencode/test/altimate/cte-detect.test.ts b/packages/opencode/test/altimate/cte-detect.test.ts index b50e6ccfdc..533c23c550 100644 --- a/packages/opencode/test/altimate/cte-detect.test.ts +++ b/packages/opencode/test/altimate/cte-detect.test.ts @@ -1521,6 +1521,9 @@ describe("count query edge cases", () => { expect(unclosedJinjaBlock(buildCountQuery(open, found, found[0])!)).toBe("if") expect(unclosedJinjaBlock("{% set q %}x")).toBe("set") expect(unclosedJinjaBlock("{% raw %}{% if x %}{% endraw %}")).toBeUndefined() + expect(unclosedJinjaBlock("{% block body %}with a as (select 1)")).toBe("block") + expect(unclosedJinjaBlock("{% if x %}a{%+ endif %}")).toBeUndefined() + expect(unclosedJinjaBlock("{% set (a, b) = (1, 2) %}x")).toBeUndefined() }) it("matches nested set blocks to their own end tag", () => { From 180ee302d611519a7a7270673defc8b878b00fd5 Mon Sep 17 00:00:00 2001 From: Harish Kumar Date: Sun, 15 Nov 2026 22:19:31 +0530 Subject: [PATCH 08/26] fix: dbt_cte_profile reads a bracket name with ]], skips a captured set block between CTEs, and matches a CTE name by case when position is given Co-Authored-By: Claude Sonnet 5.5 --- .../opencode/src/altimate/tools/cte-detect.ts | 12 ++------ .../src/altimate/tools/dbt-cte-profile.ts | 2 +- .../opencode/test/altimate/cte-detect.test.ts | 28 +++++++++++-------- .../test/altimate/dbt-cte-profile.test.ts | 6 ++-- 4 files changed, 25 insertions(+), 23 deletions(-) diff --git a/packages/opencode/src/altimate/tools/cte-detect.ts b/packages/opencode/src/altimate/tools/cte-detect.ts index 4935df22bb..7a271cab96 100644 --- a/packages/opencode/src/altimate/tools/cte-detect.ts +++ b/packages/opencode/src/altimate/tools/cte-detect.ts @@ -21,8 +21,6 @@ export interface CteInfo { index: number /** Offset of the WITH keyword that opens this CTE's clause. */ withClauseStart: number - /** The clause is `WITH RECURSIVE`. */ - recursive?: boolean } /** Position just past the string literal, quoted identifier or comment (`--`, block, Jinja) that starts at `pos`; `pos` itself when none does. An unterminated one runs to the end of the text. */ @@ -68,9 +66,7 @@ function skipTrivia(text: string, pos: number, jinja: boolean): number { continue } if (jinja && text[pos] === "{" && text[pos + 1] === "%") { - const close = text.indexOf("%}", pos + 2) - if (close === -1) return text.length - pos = close + 2 + pos = skipJinjaBlock(text, pos) continue } return pos @@ -219,7 +215,7 @@ function readIdentifier(text: string, pos: number): number { if (close === end || text[close - 1] !== char || close - end < 2) return -1 end = close } else if (char === "[") { - const bracketed = /\[[^\]\n]{1,256}\]/y + const bracketed = /\[(?:[^\]\n]|\]\]){1,256}\]/y bracketed.lastIndex = end if (!bracketed.test(text)) return -1 end = bracketed.lastIndex @@ -256,8 +252,7 @@ function parseWithClause(text: string, withPos: number, ctes: CteInfo[]): void { let pos = skipTrivia(text, withPos + 4, true) const keyword = /recursive\b/iy keyword.lastIndex = pos - const recursive = keyword.test(text) - if (recursive) pos = skipTrivia(text, pos + "recursive".length, true) + if (keyword.test(text)) pos = skipTrivia(text, pos + "recursive".length, true) for (let index = 0; ; index++) { const nameStart = pos @@ -287,7 +282,6 @@ function parseWithClause(text: string, withPos: number, ctes: CteInfo[]): void { queryEnd: close, index, withClauseStart: withPos, - ...(recursive && { recursive }), }) pos = skipTrivia(text, close + 1, true) if (text[pos] !== ",") return diff --git a/packages/opencode/src/altimate/tools/dbt-cte-profile.ts b/packages/opencode/src/altimate/tools/dbt-cte-profile.ts index ade5450efa..bb23811d50 100644 --- a/packages/opencode/src/altimate/tools/dbt-cte-profile.ts +++ b/packages/opencode/src/altimate/tools/dbt-cte-profile.ts @@ -172,7 +172,7 @@ export const DbtCteProfileTool = Tool.define("dbt_cte_profile", { if (args.position !== undefined) { target = ctes[args.position] if (!target) return failure("CTE not found", `${args.model} has ${ctes.length} CTEs: there is none at position ${args.position}.`, { model: args.model }) - if (args.cte !== undefined && bareName(target.name) !== bareName(args.cte)) + if (args.cte !== undefined && bareName(target.name) !== bareName(args.cte) && bareName(target.name).toLowerCase() !== bareName(args.cte).toLowerCase()) return failure("CTE not found", `The CTE at position ${args.position} of ${args.model} is ${stripColumnList(target.name)}, not ${args.cte}.`, { model: args.model }) } else { // The name as typed wins; letter case only decides when nothing matches it exactly. diff --git a/packages/opencode/test/altimate/cte-detect.test.ts b/packages/opencode/test/altimate/cte-detect.test.ts index 533c23c550..489f510eef 100644 --- a/packages/opencode/test/altimate/cte-detect.test.ts +++ b/packages/opencode/test/altimate/cte-detect.test.ts @@ -587,11 +587,11 @@ select * from renamed` }) }) - describe("MAX_QUOTED_IDENTIFIER_LENGTH Edge Cases", () => { - it("should handle comments with quoted strings longer than MAX_QUOTED_IDENTIFIER_LENGTH", () => { + describe("Long quoted strings inside comments", () => { + it("should ignore a long quoted string inside a comment", () => { // Create a quoted string in comment that's longer than 500 chars const veryLongQuotedString = '"' + "a".repeat(600) + '"' - const commentWithLongQuotedString = `/* This comment contains a very long quoted string: ${veryLongQuotedString} that exceeds the MAX_QUOTED_IDENTIFIER_LENGTH limit */` + const commentWithLongQuotedString = `/* This comment contains a very long quoted string: ${veryLongQuotedString} that is long */` const sql = `with ${commentWithLongQuotedString} "real_cte" as (select 1 as id) select * from "real_cte"` const document = sql @@ -603,11 +603,11 @@ select * from renamed` ctes[0], '"real_cte"', 0, - "CTE with comment containing quoted string longer than MAX_QUOTED_IDENTIFIER_LENGTH", + "CTE with comment containing quoted string longer than 500 characters", ) }) - it("should handle quoted CTE name approaching MAX_QUOTED_IDENTIFIER_LENGTH limit", () => { + it("should handle quoted CTE name approaching 500 characters", () => { // Create a quoted CTE name that's close to but under the 500 char limit const longCTEName = '"' + "very_long_cte_name_" + "x".repeat(450) + '"' @@ -617,10 +617,10 @@ select * from renamed` const ctes = detectCtes(document) expect(ctes).toHaveLength(1) - assertCTE(ctes[0], longCTEName, 0, "CTE with quoted name approaching MAX_QUOTED_IDENTIFIER_LENGTH limit") + assertCTE(ctes[0], longCTEName, 0, "CTE with quoted name approaching 500 characters") }) - it("should handle quoted CTE name exactly at MAX_QUOTED_IDENTIFIER_LENGTH limit", () => { + it("should handle quoted CTE name exactly at 500 characters", () => { // Create a quoted CTE name that's exactly at the 500 char limit (498 chars inside quotes + 2 quotes = 500) const exactLimitCTEName = '"' + "x".repeat(498) + '"' @@ -630,7 +630,7 @@ select * from renamed` const ctes = detectCtes(document) expect(ctes).toHaveLength(1) - assertCTE(ctes[0], exactLimitCTEName, 0, "CTE with quoted name exactly at MAX_QUOTED_IDENTIFIER_LENGTH limit") + assertCTE(ctes[0], exactLimitCTEName, 0, "CTE with quoted name exactly at 500 characters") }) it("should handle comment with multiple long quoted strings and real CTE", () => { @@ -649,7 +649,7 @@ select * from renamed` ctes[0], '"actual_cte"', 0, - "CTE with comment containing multiple quoted strings longer than MAX_QUOTED_IDENTIFIER_LENGTH", + "CTE with comment containing multiple quoted strings longer than 500 characters", ) }) @@ -688,7 +688,7 @@ select * from renamed` ctes[0], '"real_cte"', 0, - "CTE with line comment containing quoted string longer than MAX_QUOTED_IDENTIFIER_LENGTH", + "CTE with line comment containing quoted string longer than 500 characters", ) }) @@ -707,7 +707,7 @@ select * from renamed` ctes[0], '"real_cte"', 0, - "CTE with Jinja comment containing quoted string longer than MAX_QUOTED_IDENTIFIER_LENGTH", + "CTE with Jinja comment containing quoted string longer than 500 characters", ) }) @@ -1412,6 +1412,12 @@ select * from last` expect(detectCtesInText(sql).map((c) => c.name)).toEqual(['"a""b"', "`c``d`"]) }) + it("skips a captured {% set %} block between CTEs and reads a bracket name with an escaped ]]", () => { + const sql = "with a as (select 1),\n{% set helper %}select 2{% endset %}\nb as (select 2)\nselect * from b" + expect(detectCtesInText(sql).map((c) => c.name)).toEqual(["a", "b"]) + expect(detectCtesInText("with [we]]ird] as (select 1) select * from [we]]ird]").map((c) => c.name)).toEqual(["[we]]ird]"]) + }) + it("keeps CTEs that follow a Jinja tag between them, and reads WITH RECURSIVE and MATERIALIZED", () => { const jinja = "with a as (select 1),\n{% if x %}\nb as (select 2),\n{% endif %}\nc as (select 3)\nselect * from c" expect(detectCtesInText(jinja).map((c) => c.name)).toEqual(["a", "b", "c"]) diff --git a/packages/opencode/test/altimate/dbt-cte-profile.test.ts b/packages/opencode/test/altimate/dbt-cte-profile.test.ts index ecd93780d4..0774adafd4 100644 --- a/packages/opencode/test/altimate/dbt-cte-profile.test.ts +++ b/packages/opencode/test/altimate/dbt-cte-profile.test.ts @@ -260,11 +260,13 @@ describe("dbt_cte_profile same-named CTEs", () => { const tool = await initTool(DbtCteProfileTool) const result = await tool.execute({ model: "twins", position: 2 }, ctx) expect(result.metadata.cte).toBe("keep") + const cased = await tool.execute({ model: "twins", cte: "KEEP", position: 2 }, ctx) + expect(cased.metadata.success).toBe(true) const wrong = await tool.execute({ model: "twins", cte: "foo", position: 2 }, ctx) expect(wrong.metadata.success).toBe(false) const missing = await tool.execute({ model: "twins", position: 9 }, ctx) expect(missing.metadata.success).toBe(false) - expect(queries).toHaveLength(1) + expect(queries).toHaveLength(2) }) test("a CTE of a second WITH clause has no query of its own and runs nothing", async () => { @@ -392,7 +394,7 @@ describe("dbt_cte_profile run", () => { const controller = new AbortController() controller.abort() const tool = await initTool(DbtCteProfileTool) - await expect(tool.execute({ model: "customers", cte: "orders" }, { ...ctx, abort: controller.signal })).rejects.toBeDefined() + await expect(tool.execute({ model: "customers", cte: "orders" }, { ...ctx, abort: controller.signal })).rejects.toMatchObject({ name: "AbortError" }) expect(queries).toEqual([]) }) }) From d8c34693b09dd856b0efd2c4a1924c7320346720 Mon Sep 17 00:00:00 2001 From: Harish Kumar Date: Sun, 15 Nov 2026 22:27:01 +0530 Subject: [PATCH 09/26] fix: send falsy API bodies, follow cancellation in apiRequest, check abort before a read-only refusal - apiRequest sends any body but undefined (null, false, 0 and "" were dropped) and takes an optional signal that cancels the request along with its 15 s timeout. - dbtAdapterFor throws for a cancelled call before it would refuse a read-only or plan-mode one. - Tests cover these and a session inside the adapter's own project. Co-Authored-By: Claude Opus 5.5 --- packages/opencode/src/altimate/api/client.ts | 25 ++++++-- .../src/altimate/tools/dbt-adapter-access.ts | 2 + .../test/altimate/dbt-adapter-access.test.ts | 63 +++++++++++++++++++ 3 files changed, 84 insertions(+), 6 deletions(-) diff --git a/packages/opencode/src/altimate/api/client.ts b/packages/opencode/src/altimate/api/client.ts index cb53c0ec7b..6ea7d96354 100644 --- a/packages/opencode/src/altimate/api/client.ts +++ b/packages/opencode/src/altimate/api/client.ts @@ -318,7 +318,13 @@ export namespace AltimateApi { return data.auth_token } - async function request(creds: AltimateCredentials, method: string, endpoint: string, body?: unknown) { + async function request( + creds: AltimateCredentials, + method: string, + endpoint: string, + body?: unknown, + signal?: AbortSignal, + ) { const url = `${creds.altimateUrl}${endpoint}` // altimate_change start — upstream_fix: bound every API request. Without a // signal a stalled server holds the caller indefinitely. The abort stays @@ -327,7 +333,7 @@ export namespace AltimateApi { const timeout = setTimeout(() => controller.abort(), 15_000) try { const res = await fetch(url, { - signal: controller.signal, + signal: signal ? AbortSignal.any([controller.signal, signal]) : controller.signal, method, headers: { "Content-Type": "application/json", @@ -335,7 +341,8 @@ export namespace AltimateApi { "x-tenant": creds.altimateInstanceName, ...clientHeaders(), }, - ...(body ? { body: JSON.stringify(body) } : {}), + // Any body but undefined is sent: null, false, 0 and "" are JSON values too. + ...(body !== undefined ? { body: JSON.stringify(body) } : {}), }) if (!res.ok) { throw new Error(`API ${method} ${endpoint} failed with status ${res.status}`) @@ -350,9 +357,15 @@ export namespace AltimateApi { /** * Calls an Altimate API endpoint with the stored credentials (Bearer key and x-tenant headers) and * returns the parsed JSON body. Endpoint functions in other files go through this. Fails with a - * message naming the credentials file when none are stored or they can't be read. + * message naming the credentials file when none are stored or they can't be read. `signal` (a tool + * call's `ctx.abort`) cancels the request as well as the 15 s timeout. */ - export async function apiRequest(method: string, endpoint: string, body?: unknown): Promise { + export async function apiRequest( + method: string, + endpoint: string, + body?: unknown, + signal?: AbortSignal, + ): Promise { let creds: AltimateCredentials try { creds = await getCredentials() @@ -364,7 +377,7 @@ export namespace AltimateApi { ) throw new Error(`The Altimate credentials in ${p} can't be used: ${e instanceof Error ? e.message : String(e)}`) } - return request(creds, method, endpoint, body) + return request(creds, method, endpoint, body, signal) } export async function listDatamates(creds?: AltimateCredentials) { diff --git a/packages/opencode/src/altimate/tools/dbt-adapter-access.ts b/packages/opencode/src/altimate/tools/dbt-adapter-access.ts index d73dd7e9c5..1ff9047e9a 100644 --- a/packages/opencode/src/altimate/tools/dbt-adapter-access.ts +++ b/packages/opencode/src/altimate/tools/dbt-adapter-access.ts @@ -25,6 +25,8 @@ export const dbtAdapterAccessInternals = { */ export async function dbtAdapterFor(ctx: Pick): Promise { const signal = ctx.abort + // A cancelled call answers nothing, refused or not. + signal.throwIfAborted() if (ctx.extra?.forceReadOnly === true || ctx.agent === "plan") return { refusal: "read_only", diff --git a/packages/opencode/test/altimate/dbt-adapter-access.test.ts b/packages/opencode/test/altimate/dbt-adapter-access.test.ts index b2a012ed58..75a2fed64d 100644 --- a/packages/opencode/test/altimate/dbt-adapter-access.test.ts +++ b/packages/opencode/test/altimate/dbt-adapter-access.test.ts @@ -125,6 +125,18 @@ describe("dbtAdapterFor", () => { expect(created).toBe(1) }) + test("returns the adapter to a session inside the adapter's own project", async () => { + try { + const access = await Instance.provide({ directory: project, fn: () => dbtAdapterFor(ctx) }) + expect(access.refusal).toBeUndefined() + expect(access.reason).toBeUndefined() + expect(access.adapter?.isInitialized()).toBe(true) + expect(created).toBe(1) + } finally { + await disposeInstance(project) + } + }) + test("says when the adapter didn't load the project", async () => { dbtAdapterInternals.create = async () => ({ isInitialized: () => false, dispose: async () => {} }) const access = await dbtAdapterFor(ctx) @@ -180,6 +192,15 @@ describe("dbtAdapterFor", () => { expect(created).toBe(0) }) + test("a cancelled read-only or plan-mode call throws rather than being refused", async () => { + const controller = new AbortController() + controller.abort() + const abort = controller.signal + await expect(dbtAdapterFor({ ...ctx, abort, extra: { forceReadOnly: true } })).rejects.toThrow() + await expect(dbtAdapterFor({ ...ctx, abort, agent: "plan" })).rejects.toThrow() + expect(created).toBe(0) + }) + test("a session in a dbt project the adapter isn't set up for says so", async () => { dbtAdapterInternals.readConfig = async () => null try { @@ -260,6 +281,48 @@ describe("AltimateApi.apiRequest", () => { } }) + test("sends a falsy JSON body rather than dropping it", async () => { + await Bun.write( + path.join(home, ".altimate", "altimate.json"), + JSON.stringify({ altimateUrl: "https://api.example.test", altimateInstanceName: "acme", altimateApiKey: "k1" }), + ) + const original = globalThis.fetch + const bodies: unknown[] = [] + globalThis.fetch = (async (_url: string, init: RequestInit) => { + bodies.push(init.body) + return new Response("{}", { status: 200 }) + }) as unknown as typeof fetch + try { + for (const body of [null, false, 0, ""]) await AltimateApi.apiRequest("POST", "/dbt/v1/x", body) + await AltimateApi.apiRequest("GET", "/dbt/v1/x") + expect(bodies).toEqual(["null", "false", "0", '""', undefined]) + } finally { + globalThis.fetch = original + } + }) + + test("a cancelled call stops the request", async () => { + await Bun.write( + path.join(home, ".altimate", "altimate.json"), + JSON.stringify({ altimateUrl: "https://api.example.test", altimateInstanceName: "acme", altimateApiKey: "k1" }), + ) + const original = globalThis.fetch + globalThis.fetch = ((_url: string, init: RequestInit) => + new Promise((_, reject) => { + // Like fetch: a signal already aborted rejects at once, else the abort does. + if (init.signal?.aborted) return reject(init.signal.reason) + init.signal?.addEventListener("abort", () => reject(init.signal?.reason), { once: true }) + })) as unknown as typeof fetch + const controller = new AbortController() + try { + const pending = AltimateApi.apiRequest("GET", "/dbt/v1/x", undefined, controller.signal) + controller.abort(new Error("cancelled by the user")) + await expect(pending).rejects.toThrow("cancelled by the user") + } finally { + globalThis.fetch = original + } + }) + test("names the file when the stored credentials can't be read", async () => { await Bun.write(path.join(home, ".altimate", "altimate.json"), "{not json") await expect(AltimateApi.apiRequest("GET", "/dbt/v1/x")).rejects.toThrow( From 56f100693b6d46d317a61717a33e348345f8eada Mon Sep 17 00:00:00 2001 From: Harish Kumar Date: Tue, 17 Nov 2026 06:14:43 +0530 Subject: [PATCH 10/26] fix: dbt_cte_profile undoes ]] in a bracketed CTE name when matching it Co-Authored-By: Claude Sonnet 5.5 --- .../opencode/src/altimate/tools/dbt-cte-profile.ts | 2 +- .../opencode/test/altimate/dbt-cte-profile.test.ts | 12 ++++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/packages/opencode/src/altimate/tools/dbt-cte-profile.ts b/packages/opencode/src/altimate/tools/dbt-cte-profile.ts index bb23811d50..6107fa7123 100644 --- a/packages/opencode/src/altimate/tools/dbt-cte-profile.ts +++ b/packages/opencode/src/altimate/tools/dbt-cte-profile.ts @@ -23,7 +23,7 @@ function bareName(name: string): string { const plain = stripColumnList(name) const quote = plain[0] if ((quote === '"' || quote === "`") && plain.length > 1 && plain.endsWith(quote)) return plain.slice(1, -1).replaceAll(quote + quote, quote) - if (quote === "[" && plain.endsWith("]")) return plain.slice(1, -1) + if (quote === "[" && plain.endsWith("]")) return plain.slice(1, -1).replaceAll("]]", "]") return plain } diff --git a/packages/opencode/test/altimate/dbt-cte-profile.test.ts b/packages/opencode/test/altimate/dbt-cte-profile.test.ts index 0774adafd4..f019336bf1 100644 --- a/packages/opencode/test/altimate/dbt-cte-profile.test.ts +++ b/packages/opencode/test/altimate/dbt-cte-profile.test.ts @@ -74,6 +74,7 @@ beforeAll(async () => { path.join(project, "models", "marts", "twins.sql"), 'with "Foo" as (select 1 as a), foo as (select 2 as a), keep as (select 3 as a)\nselect * from keep\n', ) + await writeFile(path.join(project, "models", "marts", "bracketed.sql"), "with [we]]ird] as (select 1 as a) select * from [we]]ird]\n") await writeFile( path.join(project, "models", "marts", "wrapped.sql"), "{% if is_incremental() %}\nwith a as (select 1 as x) select * from a\n{% else %}\nwith b as (select 2 as x) select * from b\n{% endif %}\n", @@ -238,6 +239,17 @@ describe("dbt_cte_profile model lookup", () => { }) }) +describe("dbt_cte_profile bracket-quoted names", () => { + test("a ]] inside a bracketed name is one ], so the CTE is reachable by its plain name", async () => { + const tool = await initTool(DbtCteProfileTool) + for (const cte of ["we]ird", "[we]]ird]"]) { + const run = await tool.execute({ model: "bracketed", cte }, ctx) + expect(run.metadata.success).toBe(true) + expect(run.metadata.position).toBe(0) + } + }) +}) + describe("dbt_cte_profile same-named CTEs", () => { test("two CTEs whose names differ only in case are listed apart, and each is reachable by its exact name", async () => { const tool = await initTool(DbtCteProfileTool) From 18a3fc4e0bd124e8f5fb3120fdc4d152be88cbe3 Mon Sep 17 00:00:00 2001 From: Harish Kumar Date: Tue, 17 Nov 2026 06:39:46 +0530 Subject: [PATCH 11/26] fix: keep the stored API key out of API errors, bound the dbt config read - apiRequest names what is wrong with the credentials file (not valid JSON, which field is missing) instead of quoting the parser's message, which can quote the key. - request() redacts the key from a failed send: Bun's header check quotes `Bearer ` for a key with CR/LF, and throws before fetch returns a promise, so both a throw and a rejection are caught. - A request that times out says so; a cancelled apiRequest reads nothing. - dbtAdapterFor bounds the project lookup with the same deadline and abort as the adapter's start, and checks the abort after each wait. Co-Authored-By: Claude Opus 5.5 --- packages/opencode/src/altimate/api/client.ts | 75 ++++++-- .../src/altimate/tools/dbt-adapter-access.ts | 52 ++--- .../test/altimate/dbt-adapter-access.test.ts | 179 +++++++++++++++++- 3 files changed, 265 insertions(+), 41 deletions(-) diff --git a/packages/opencode/src/altimate/api/client.ts b/packages/opencode/src/altimate/api/client.ts index 6ea7d96354..d3cfd1a887 100644 --- a/packages/opencode/src/altimate/api/client.ts +++ b/packages/opencode/src/altimate/api/client.ts @@ -318,6 +318,17 @@ export namespace AltimateApi { return data.auth_token } + /** Test seams. */ + export const requestInternals = { + /** How long an API request may take, body included, before it is aborted. */ + timeoutMs: 15_000, + } + + /** `text` with every occurrence of `secret` replaced, so an error can be shown without the key. */ + function redact(text: string, secret: string): string { + return secret ? text.split(secret).join("[redacted]") : text + } + async function request( creds: AltimateCredentials, method: string, @@ -326,24 +337,43 @@ export namespace AltimateApi { signal?: AbortSignal, ) { const url = `${creds.altimateUrl}${endpoint}` + // Bun's fetch throws at once, not as a rejection, when it refuses a header: both are caught. + const send = async (start: () => Promise) => { + try { + return await start() + } catch (e) { + // A timeout or the caller's abort rejects with its own reason, which holds no header. + if (controller.signal.aborted || signal?.aborted) throw e + // Others can quote a header: Bun's check of a header value with CR/LF quotes `Bearer `. + const message = redact(e instanceof Error ? e.message : String(e), creds.altimateApiKey) + throw new Error(`API ${method} ${endpoint} could not be sent: ${message}`) + } + } // altimate_change start — upstream_fix: bound every API request. Without a // signal a stalled server holds the caller indefinitely. The abort stays // armed until the BODY is read: `fetch` resolves on headers. const controller = new AbortController() - const timeout = setTimeout(() => controller.abort(), 15_000) + const seconds = requestInternals.timeoutMs / 1000 + const timeout = setTimeout( + () => + controller.abort(new DOMException(`API ${method} ${endpoint} timed out after ${seconds} s`, "TimeoutError")), + requestInternals.timeoutMs, + ) try { - const res = await fetch(url, { - signal: signal ? AbortSignal.any([controller.signal, signal]) : controller.signal, - method, - headers: { - "Content-Type": "application/json", - Authorization: `Bearer ${creds.altimateApiKey}`, - "x-tenant": creds.altimateInstanceName, - ...clientHeaders(), - }, - // Any body but undefined is sent: null, false, 0 and "" are JSON values too. - ...(body !== undefined ? { body: JSON.stringify(body) } : {}), - }) + const res = await send(() => + fetch(url, { + signal: signal ? AbortSignal.any([controller.signal, signal]) : controller.signal, + method, + headers: { + "Content-Type": "application/json", + Authorization: `Bearer ${creds.altimateApiKey}`, + "x-tenant": creds.altimateInstanceName, + ...clientHeaders(), + }, + // Any body but undefined is sent: null, false, 0 and "" are JSON values too. + ...(body !== undefined ? { body: JSON.stringify(body) } : {}), + }), + ) if (!res.ok) { throw new Error(`API ${method} ${endpoint} failed with status ${res.status}`) } @@ -354,11 +384,25 @@ export namespace AltimateApi { // altimate_change end } + /** What is wrong with the stored credentials, naming fields but never quoting the file's text. */ + function credentialsProblem(e: unknown): string { + // The parser's message quotes the text it stopped at, which can be the key. + if (e instanceof SyntaxError) return "the file is not valid JSON." + if (e instanceof z.ZodError) { + const fields = [...new Set(e.issues.map((issue) => issue.path.join(".")))] + if (fields.length === 0 || fields.includes("")) return "the file does not hold the credentials object." + return `${fields.join(", ")} ${fields.length === 1 ? "is" : "are"} missing or not valid.` + } + // A file system error or a missing ${env:NAME} variable: names a path or a variable, no value. + return e instanceof Error ? e.message : String(e) + } + /** * Calls an Altimate API endpoint with the stored credentials (Bearer key and x-tenant headers) and * returns the parsed JSON body. Endpoint functions in other files go through this. Fails with a * message naming the credentials file when none are stored or they can't be read. `signal` (a tool - * call's `ctx.abort`) cancels the request as well as the 15 s timeout. + * call's `ctx.abort`) cancels the request as well as the 15 s timeout. No error it raises quotes the + * stored key. */ export async function apiRequest( method: string, @@ -366,6 +410,7 @@ export namespace AltimateApi { body?: unknown, signal?: AbortSignal, ): Promise { + signal?.throwIfAborted() let creds: AltimateCredentials try { creds = await getCredentials() @@ -375,7 +420,7 @@ export namespace AltimateApi { throw new Error( `No Altimate credentials are stored: sign in to Altimate in altimate-code first (they are saved to ${p}).`, ) - throw new Error(`The Altimate credentials in ${p} can't be used: ${e instanceof Error ? e.message : String(e)}`) + throw new Error(`The Altimate credentials in ${p} can't be used: ${credentialsProblem(e)}`) } return request(creds, method, endpoint, body, signal) } diff --git a/packages/opencode/src/altimate/tools/dbt-adapter-access.ts b/packages/opencode/src/altimate/tools/dbt-adapter-access.ts index 1ff9047e9a..df8fb597dc 100644 --- a/packages/opencode/src/altimate/tools/dbt-adapter-access.ts +++ b/packages/opencode/src/altimate/tools/dbt-adapter-access.ts @@ -21,7 +21,8 @@ export const dbtAdapterAccessInternals = { * The dbt adapter for a tool call, which every adapter-backed tool goes through. Starting the adapter * runs dbt deps and writes target/, and compiling renders Jinja, which can run SQL: a read-only call * (no write permission, or plan mode) never starts or uses it. Nor does a session in a dbt project the - * adapter isn't set up for. The wait for the adapter to start is bounded and follows the call's abort. + * adapter isn't set up for. Every wait, the project lookup and the adapter's start, is bounded by one + * deadline and follows the call's abort. */ export async function dbtAdapterFor(ctx: Pick): Promise { const signal = ctx.abort @@ -32,12 +33,30 @@ export async function dbtAdapterFor(ctx: Pick {} + let timer: ReturnType | undefined + const waited = new Promise((_, reject) => { + stop = () => reject(signal.reason) + signal.addEventListener("abort", stop, { once: true }) + const seconds = dbtAdapterAccessInternals.adapterStartTimeoutMs / 1000 + timer = setTimeout( + () => reject(new Error(`the dbt adapter didn't start within ${seconds} s`)), + dbtAdapterAccessInternals.adapterStartTimeoutMs, + ) + }) + // It can reject between two races, when nothing awaits it: the next race still sees the rejection. + waited.catch(() => {}) + const bounded = async (work: Promise): Promise => { + const value = await Promise.race([work, waited]) + // An abort queued while the work settled still wins. signal.throwIfAborted() + return value + } + try { + const { getDbtAdapter, getDbtProjectRoot } = await bounded(import("../native/connections/register")) + const project = await bounded(sessionProject()) if (project?.other) { return { refusal: "other_project", @@ -46,24 +65,10 @@ export async function dbtAdapterFor(ctx: Pick {} - let timer: ReturnType | undefined - const waited = new Promise((_, reject) => { - stop = () => reject(signal.reason) - signal.addEventListener("abort", stop, { once: true }) - const seconds = dbtAdapterAccessInternals.adapterStartTimeoutMs / 1000 - timer = setTimeout( - () => reject(new Error(`the dbt adapter didn't start within ${seconds} s`)), - dbtAdapterAccessInternals.adapterStartTimeoutMs, - ) - }) - const adapter = await Promise.race([getDbtAdapter(), waited]).finally(() => { - signal.removeEventListener("abort", stop) - clearTimeout(timer) - }) + const adapter = await bounded(getDbtAdapter()) if (!adapter) { // No adapter also when the project exists but the adapter failed to start: say which. - const root = await getDbtProjectRoot() + const root = await bounded(getDbtProjectRoot()) const found = !!root && fs.existsSync(path.join(root, "dbt_project.yml")) return found ? { refusal: "not_started", reason: "the dbt adapter didn't start" } @@ -75,6 +80,9 @@ export async function dbtAdapterFor(ctx: Pick { } }) + test("a stalled dbt config read gives up at the start timeout", async () => { + let release = () => {} + dbtAdapterInternals.readConfig = () => new Promise((resolve) => (release = () => resolve(null))) + const saved = dbtAdapterAccessInternals.adapterStartTimeoutMs + dbtAdapterAccessInternals.adapterStartTimeoutMs = 20 + try { + const access = await Instance.provide({ directory: project, fn: () => dbtAdapterFor(ctx) }) + expect(access).toEqual({ refusal: "not_started", reason: "the dbt adapter didn't start within 0.02 s" }) + expect(created).toBe(0) + } finally { + release() + dbtAdapterAccessInternals.adapterStartTimeoutMs = saved + await disposeInstance(project) + } + }) + + test("a call cancelled during a stalled dbt config read throws at once", async () => { + let release = () => {} + let entered = () => {} + const reading = new Promise((resolve) => (entered = resolve)) + dbtAdapterInternals.readConfig = () => { + entered() + return new Promise((resolve) => (release = () => resolve(null))) + } + const controller = new AbortController() + try { + const pending = Instance.provide({ + directory: project, + fn: () => dbtAdapterFor({ ...ctx, abort: controller.signal }), + }) + await reading + controller.abort(new Error("cancelled")) + await expect(pending).rejects.toThrow("cancelled") + } finally { + release() + await disposeInstance(project) + } + }) + + test("removes its abort listener and its timer whatever the outcome", async () => { + const controller = new AbortController() + const signal = controller.signal + let listening = 0 + const add = signal.addEventListener.bind(signal) + const remove = signal.removeEventListener.bind(signal) + signal.addEventListener = ((...a: Parameters) => { + if (a[0] === "abort") listening += 1 + return add(...a) + }) as typeof signal.addEventListener + signal.removeEventListener = ((...a: Parameters) => { + if (a[0] === "abort") listening -= 1 + return remove(...a) + }) as typeof signal.removeEventListener + const timers = new Set() + const set = globalThis.setTimeout + const clear = globalThis.clearTimeout + globalThis.setTimeout = ((fn: () => void, ms?: number) => { + const t = set(fn, ms) + timers.add(t) + return t + }) as typeof setTimeout + globalThis.clearTimeout = ((t: ReturnType) => { + timers.delete(t) + return clear(t) + }) as typeof clearTimeout + try { + expect((await dbtAdapterFor({ ...ctx, abort: signal })).adapter).toBeDefined() + dbtAdapterInternals.create = async () => { + throw new Error("dbt-integration missing") + } + resetDbtAdapter() + expect((await dbtAdapterFor({ ...ctx, abort: signal })).refusal).toBe("not_started") + } finally { + globalThis.setTimeout = set + globalThis.clearTimeout = clear + } + expect(listening).toBe(0) + expect(timers.size).toBe(0) + }) + test("a session in another dbt project than the adapter's says which", async () => { const other = await mkdtemp(path.join(os.tmpdir(), "dbt-adapter-other-")) await writeFile(path.join(other, "dbt_project.yml"), "name: other\n") @@ -323,10 +403,101 @@ describe("AltimateApi.apiRequest", () => { } }) - test("names the file when the stored credentials can't be read", async () => { - await Bun.write(path.join(home, ".altimate", "altimate.json"), "{not json") - await expect(AltimateApi.apiRequest("GET", "/dbt/v1/x")).rejects.toThrow( - `The Altimate credentials in ${path.join(home, ".altimate", "altimate.json")} can't be used`, + test("names the file and the failure, never the file's text, when the credentials aren't valid JSON", async () => { + const file = path.join(home, ".altimate", "altimate.json") + await Bun.write( + file, + '{"altimateUrl":"https://x.test","altimateInstanceName":"acme","altimateApiKey": sk_SECRET123}', + ) + const error = await AltimateApi.apiRequest("GET", "/dbt/v1/x").catch((e: Error) => e) + expect(error).toBeInstanceOf(Error) + expect(error.message).toBe(`The Altimate credentials in ${file} can't be used: the file is not valid JSON.`) + expect(error.message).not.toContain("sk_SECRET123") + }) + + test("names the missing field, never a value, when the credentials are incomplete", async () => { + const file = path.join(home, ".altimate", "altimate.json") + await Bun.write(file, JSON.stringify({ altimateUrl: "https://x.test", altimateApiKey: "sk_SECRET123" })) + const error = await AltimateApi.apiRequest("GET", "/dbt/v1/x").catch((e: Error) => e) + expect(error.message).toBe( + `The Altimate credentials in ${file} can't be used: altimateInstanceName is missing or not valid.`, ) + expect(error.message).not.toContain("sk_SECRET123") + }) + + test("never shows the stored key when sending the request fails", async () => { + const key = "sk_SECRET123\r\nX: y" + await Bun.write( + path.join(home, ".altimate", "altimate.json"), + JSON.stringify({ altimateUrl: "http://127.0.0.1:9", altimateInstanceName: "acme", altimateApiKey: key }), + ) + // The real fetch: it rejects a header value with CR/LF, quoting the value. + const error = await AltimateApi.apiRequest("GET", "/dbt/v1/x").catch((e: Error) => e) + expect(error).toBeInstanceOf(Error) + expect(error.message).not.toContain("sk_SECRET123") + expect(error.message).toContain("API GET /dbt/v1/x could not be sent") + }) + + test("never shows the stored key from any fetch failure", async () => { + await Bun.write( + path.join(home, ".altimate", "altimate.json"), + JSON.stringify({ + altimateUrl: "https://api.example.test", + altimateInstanceName: "acme", + altimateApiKey: "sk_SECRET123", + }), + ) + const original = globalThis.fetch + globalThis.fetch = (async (_url: string, init: RequestInit) => { + throw new TypeError(`bad header: '${(init.headers as Record).Authorization}'`) + }) as unknown as typeof fetch + try { + const error = await AltimateApi.apiRequest("GET", "/dbt/v1/x").catch((e: Error) => e) + expect(error.message).toBe("API GET /dbt/v1/x could not be sent: bad header: 'Bearer [redacted]'") + } finally { + globalThis.fetch = original + } + }) + + test("a timed-out request says it timed out", async () => { + await Bun.write( + path.join(home, ".altimate", "altimate.json"), + JSON.stringify({ altimateUrl: "https://api.example.test", altimateInstanceName: "acme", altimateApiKey: "k1" }), + ) + const original = globalThis.fetch + globalThis.fetch = ((_url: string, init: RequestInit) => + new Promise((_, reject) => { + init.signal?.addEventListener("abort", () => reject(init.signal?.reason), { once: true }) + })) as unknown as typeof fetch + const saved = AltimateApi.requestInternals.timeoutMs + AltimateApi.requestInternals.timeoutMs = 10 + try { + await expect(AltimateApi.apiRequest("GET", "/dbt/v1/x")).rejects.toThrow( + "API GET /dbt/v1/x timed out after 0.01 s", + ) + } finally { + AltimateApi.requestInternals.timeoutMs = saved + globalThis.fetch = original + } + }) + + test("a call cancelled before it starts reads no credentials and sends nothing", async () => { + const original = globalThis.fetch + let sent = 0 + globalThis.fetch = (async () => { + sent += 1 + return new Response("{}") + }) as unknown as typeof fetch + const controller = new AbortController() + controller.abort(new Error("cancelled by the user")) + try { + // No credentials are stored: reading them would fail with another message. + await expect(AltimateApi.apiRequest("GET", "/dbt/v1/x", undefined, controller.signal)).rejects.toThrow( + "cancelled by the user", + ) + expect(sent).toBe(0) + } finally { + globalThis.fetch = original + } }) }) From 6024ac9b208400ff2c70f8ef07e98b1568163276 Mon Sep 17 00:00:00 2001 From: Harish Kumar Date: Tue, 17 Nov 2026 06:47:20 +0530 Subject: [PATCH 12/26] fix: redact the API key when a refused header races the caller's abort; assert stub failure classification Co-Authored-By: Claude Opus 5.5 --- packages/opencode/src/altimate/api/client.ts | 5 ++-- .../test/altimate/dbt-adapter-access.test.ts | 27 +++++++++++++++++++ 2 files changed, 30 insertions(+), 2 deletions(-) diff --git a/packages/opencode/src/altimate/api/client.ts b/packages/opencode/src/altimate/api/client.ts index d3cfd1a887..25a78d8bd0 100644 --- a/packages/opencode/src/altimate/api/client.ts +++ b/packages/opencode/src/altimate/api/client.ts @@ -342,8 +342,9 @@ export namespace AltimateApi { try { return await start() } catch (e) { - // A timeout or the caller's abort rejects with its own reason, which holds no header. - if (controller.signal.aborted || signal?.aborted) throw e + // A timeout or the caller's abort rejects with its own reason, which holds no header. An abort + // that raced a refused header leaves the header error, so match the reason, not the flag. + if (e === controller.signal.reason || e === signal?.reason) throw e // Others can quote a header: Bun's check of a header value with CR/LF quotes `Bearer `. const message = redact(e instanceof Error ? e.message : String(e), creds.altimateApiKey) throw new Error(`API ${method} ${endpoint} could not be sent: ${message}`) diff --git a/packages/opencode/test/altimate/dbt-adapter-access.test.ts b/packages/opencode/test/altimate/dbt-adapter-access.test.ts index 53592c8bc6..f998a83334 100644 --- a/packages/opencode/test/altimate/dbt-adapter-access.test.ts +++ b/packages/opencode/test/altimate/dbt-adapter-access.test.ts @@ -11,6 +11,7 @@ import type { Tool } from "@/tool/tool" import { SessionID, MessageID } from "@/session/schema" import { dbtAdapterInternals, resetDbtAdapter } from "@/altimate/native/connections/register" import { dbtAdapterAccessInternals, dbtAdapterFor } from "@/altimate/tools/dbt-adapter-access" +import { resultFailed } from "@/altimate/mcp-server/adapter" import { EXPOSED, MCP_OPEN_WRITE_TOOLS, WRITE_TOOLS } from "@/altimate/tool-cli/allow-list" import { AltimateApi } from "@/altimate/api/client" import { Instance } from "@/project/instance" @@ -89,6 +90,7 @@ describe("feature tools in the allow-list", () => { expect(result.title).toBe(`${id}: not implemented`) expect(result.metadata).toEqual({ success: false, error: "not implemented", truncated: false }) expect(result.output).toContain(id) + expect(resultFailed(result)).toBe(true) } }) }) @@ -438,6 +440,31 @@ describe("AltimateApi.apiRequest", () => { expect(error.message).toContain("API GET /dbt/v1/x could not be sent") }) + test("never shows the stored key when a header refusal races the caller's abort", async () => { + await Bun.write( + path.join(home, ".altimate", "altimate.json"), + JSON.stringify({ + altimateUrl: "https://api.example.test", + altimateInstanceName: "acme", + altimateApiKey: "sk_SECRET123", + }), + ) + const original = globalThis.fetch + const controller = new AbortController() + globalThis.fetch = (async (_url: string, init: RequestInit) => { + controller.abort(new Error("cancelled by the user")) + throw new TypeError(`bad header: '${(init.headers as Record).Authorization}'`) + }) as unknown as typeof fetch + try { + const error = await AltimateApi.apiRequest("GET", "/dbt/v1/x", undefined, controller.signal).catch( + (e: Error) => e, + ) + expect(error.message).toBe("API GET /dbt/v1/x could not be sent: bad header: 'Bearer [redacted]'") + } finally { + globalThis.fetch = original + } + }) + test("never shows the stored key from any fetch failure", async () => { await Bun.write( path.join(home, ".altimate", "altimate.json"), From f5159c51d1fcc7b8024a6d475ecc39106bbe8371 Mon Sep 17 00:00:00 2001 From: Harish Kumar Date: Tue, 17 Nov 2026 06:47:45 +0530 Subject: [PATCH 13/26] docs: say which tools use the dbt adapter helper Co-Authored-By: Claude Opus 5.5 --- packages/opencode/src/altimate/tools/dbt-adapter-access.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/opencode/src/altimate/tools/dbt-adapter-access.ts b/packages/opencode/src/altimate/tools/dbt-adapter-access.ts index df8fb597dc..7f93d822e9 100644 --- a/packages/opencode/src/altimate/tools/dbt-adapter-access.ts +++ b/packages/opencode/src/altimate/tools/dbt-adapter-access.ts @@ -18,7 +18,7 @@ export const dbtAdapterAccessInternals = { } /** - * The dbt adapter for a tool call, which every adapter-backed tool goes through. Starting the adapter + * The dbt adapter for a tool call, which the adapter-backed tools added after dbt_lineage go through. Starting the adapter * runs dbt deps and writes target/, and compiling renders Jinja, which can run SQL: a read-only call * (no write permission, or plan mode) never starts or uses it. Nor does a session in a dbt project the * adapter isn't set up for. Every wait, the project lookup and the adapter's start, is bounded by one From f57de3da23aab9065822038372fbb2eb5e120299 Mon Sep 17 00:00:00 2001 From: Harish Kumar Date: Tue, 17 Nov 2026 06:52:44 +0530 Subject: [PATCH 14/26] docs: rewrap and correct the adapter helper comments Co-Authored-By: Claude Opus 5.5 --- .../opencode/src/altimate/tools/dbt-adapter-access.ts | 10 +++++----- .../opencode/test/altimate/dbt-adapter-access.test.ts | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/packages/opencode/src/altimate/tools/dbt-adapter-access.ts b/packages/opencode/src/altimate/tools/dbt-adapter-access.ts index 7f93d822e9..2fc2aba423 100644 --- a/packages/opencode/src/altimate/tools/dbt-adapter-access.ts +++ b/packages/opencode/src/altimate/tools/dbt-adapter-access.ts @@ -18,11 +18,11 @@ export const dbtAdapterAccessInternals = { } /** - * The dbt adapter for a tool call, which the adapter-backed tools added after dbt_lineage go through. Starting the adapter - * runs dbt deps and writes target/, and compiling renders Jinja, which can run SQL: a read-only call - * (no write permission, or plan mode) never starts or uses it. Nor does a session in a dbt project the - * adapter isn't set up for. Every wait, the project lookup and the adapter's start, is bounded by one - * deadline and follows the call's abort. + * The dbt adapter for a tool call, which the adapter-backed tools added after dbt_lineage go through. + * Starting the adapter runs dbt deps and writes target/, and compiling renders Jinja, which can run + * SQL: a read-only call (no write permission, or plan mode) never starts or uses it. Nor does a + * session in a dbt project the adapter isn't set up for. Every wait, the project lookup and the + * adapter's start, is bounded by one deadline and follows the call's abort. */ export async function dbtAdapterFor(ctx: Pick): Promise { const signal = ctx.abort diff --git a/packages/opencode/test/altimate/dbt-adapter-access.test.ts b/packages/opencode/test/altimate/dbt-adapter-access.test.ts index f998a83334..ddcf0ea183 100644 --- a/packages/opencode/test/altimate/dbt-adapter-access.test.ts +++ b/packages/opencode/test/altimate/dbt-adapter-access.test.ts @@ -1,5 +1,5 @@ // The tools the dbt Power User features call: their allow-list entries and placeholder answers, the -// one gate every dbt-adapter-backed tool goes through, and the API helper for new endpoint files. The +// one gate the new dbt-adapter-backed tools go through, and the API helper for new endpoint files. The // dbt adapter is replaced through `dbtAdapterInternals`, so these tests need no Python or dbt. import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, test } from "bun:test" import { mkdtemp, rm, writeFile } from "node:fs/promises" From e0c5a20bba2d43ea9c1215969061f82b7de6d484 Mon Sep 17 00:00:00 2001 From: Harish Kumar Date: Tue, 17 Nov 2026 07:51:41 +0530 Subject: [PATCH 15/26] fix: dbt_cte_profile skips captured Jinja blocks inside a CTE body and prefers the exact name when position is also given Co-Authored-By: Claude Sonnet 5.5 --- packages/opencode/src/altimate/tools/cte-detect.ts | 5 +++++ packages/opencode/src/altimate/tools/dbt-cte-profile.ts | 5 ++++- packages/opencode/test/altimate/cte-detect.test.ts | 5 +++++ packages/opencode/test/altimate/dbt-cte-profile.test.ts | 6 +++++- 4 files changed, 19 insertions(+), 2 deletions(-) diff --git a/packages/opencode/src/altimate/tools/cte-detect.ts b/packages/opencode/src/altimate/tools/cte-detect.ts index 7a271cab96..734defdec5 100644 --- a/packages/opencode/src/altimate/tools/cte-detect.ts +++ b/packages/opencode/src/altimate/tools/cte-detect.ts @@ -84,6 +84,11 @@ function findMatchingClosingParen(text: string, openParenPos: number): number { pos = end continue } + // A captured block (`{% set x %}…{% endset %}`) is not the CTE's SQL: a `)` written there does not close it. + if (text[pos] === "{" && text[pos + 1] === "%") { + pos = skipJinjaBlock(text, pos) + continue + } if (text[pos] === "(") depth++ else if (text[pos] === ")" && --depth === 0) return pos pos++ diff --git a/packages/opencode/src/altimate/tools/dbt-cte-profile.ts b/packages/opencode/src/altimate/tools/dbt-cte-profile.ts index 6107fa7123..bb005b800d 100644 --- a/packages/opencode/src/altimate/tools/dbt-cte-profile.ts +++ b/packages/opencode/src/altimate/tools/dbt-cte-profile.ts @@ -172,7 +172,10 @@ export const DbtCteProfileTool = Tool.define("dbt_cte_profile", { if (args.position !== undefined) { target = ctes[args.position] if (!target) return failure("CTE not found", `${args.model} has ${ctes.length} CTEs: there is none at position ${args.position}.`, { model: args.model }) - if (args.cte !== undefined && bareName(target.name) !== bareName(args.cte) && bareName(target.name).toLowerCase() !== bareName(args.cte).toLowerCase()) + // The name as typed wins here too: another CTE spelled exactly so is the one meant, whatever the case of this one. + const typed = args.cte === undefined ? undefined : bareName(args.cte) + const spelledSo = typed === undefined ? false : ctes.some((c) => bareName(c.name) === typed) + if (typed !== undefined && (spelledSo ? bareName(target.name) !== typed : bareName(target.name).toLowerCase() !== typed.toLowerCase())) return failure("CTE not found", `The CTE at position ${args.position} of ${args.model} is ${stripColumnList(target.name)}, not ${args.cte}.`, { model: args.model }) } else { // The name as typed wins; letter case only decides when nothing matches it exactly. diff --git a/packages/opencode/test/altimate/cte-detect.test.ts b/packages/opencode/test/altimate/cte-detect.test.ts index 489f510eef..f4adbd2346 100644 --- a/packages/opencode/test/altimate/cte-detect.test.ts +++ b/packages/opencode/test/altimate/cte-detect.test.ts @@ -1418,6 +1418,11 @@ select * from last` expect(detectCtesInText("with [we]]ird] as (select 1) select * from [we]]ird]").map((c) => c.name)).toEqual(["[we]]ird]"]) }) + it("a ) inside a captured {% set %} block does not close the CTE body around it", () => { + const sql = "with a as (\n{% set helper %}){% endset %}\nselect 1), b as (select 2)\nselect * from b" + expect(detectCtesInText(sql).map((c) => c.name)).toEqual(["a", "b"]) + }) + it("keeps CTEs that follow a Jinja tag between them, and reads WITH RECURSIVE and MATERIALIZED", () => { const jinja = "with a as (select 1),\n{% if x %}\nb as (select 2),\n{% endif %}\nc as (select 3)\nselect * from c" expect(detectCtesInText(jinja).map((c) => c.name)).toEqual(["a", "b", "c"]) diff --git a/packages/opencode/test/altimate/dbt-cte-profile.test.ts b/packages/opencode/test/altimate/dbt-cte-profile.test.ts index f019336bf1..60f33384cf 100644 --- a/packages/opencode/test/altimate/dbt-cte-profile.test.ts +++ b/packages/opencode/test/altimate/dbt-cte-profile.test.ts @@ -276,9 +276,13 @@ describe("dbt_cte_profile same-named CTEs", () => { expect(cased.metadata.success).toBe(true) const wrong = await tool.execute({ model: "twins", cte: "foo", position: 2 }, ctx) expect(wrong.metadata.success).toBe(false) + const shadowed = await tool.execute({ model: "twins", cte: "foo", position: 0 }, ctx) + expect(shadowed.metadata.success).toBe(false) + const exactAt = await tool.execute({ model: "twins", cte: "foo", position: 1 }, ctx) + expect(exactAt.metadata.success).toBe(true) const missing = await tool.execute({ model: "twins", position: 9 }, ctx) expect(missing.metadata.success).toBe(false) - expect(queries).toHaveLength(2) + expect(queries).toHaveLength(3) }) test("a CTE of a second WITH clause has no query of its own and runs nothing", async () => { From 6d8243892cf9e67e4a287ff648aae7108cb69636 Mon Sep 17 00:00:00 2001 From: Harish Kumar Date: Tue, 17 Nov 2026 08:00:58 +0530 Subject: [PATCH 16/26] fix: dbt_cte_profile runs against the model's own name, numbers positions from 0, and says when a Jinja-built CTE ends the list Co-Authored-By: Claude Sonnet 5.5 --- .../opencode/src/altimate/tools/cte-detect.ts | 13 ++++++++++ .../src/altimate/tools/dbt-cte-profile.ts | 17 +++++++------ .../test/altimate/dbt-cte-profile.test.ts | 25 ++++++++++++++++++- 3 files changed, 47 insertions(+), 8 deletions(-) diff --git a/packages/opencode/src/altimate/tools/cte-detect.ts b/packages/opencode/src/altimate/tools/cte-detect.ts index 734defdec5..1b9658ac92 100644 --- a/packages/opencode/src/altimate/tools/cte-detect.ts +++ b/packages/opencode/src/altimate/tools/cte-detect.ts @@ -301,6 +301,19 @@ export function detectCtes(text: string): CteInfo[] { return ctes } +/** + * Line-independent offset of a comma that follows the last CTE read of a WITH clause: another CTE comes + * after it but could not be read (its name is built by a Jinja expression or loop), so the list is short. + */ +export function unreadCteAfter(text: string, ctes: CteInfo[]): number | undefined { + for (const c of ctes) { + if (ctes.some((other) => other.withClauseStart === c.withClauseStart && other.index === c.index + 1)) continue + const next = skipTrivia(text, c.queryEnd + 1, true) + if (text[next] === ",") return next + } + return undefined +} + /** Drop a trailing column list: `my_cte (id, name)` -> `my_cte`. */ export function stripColumnList(rawName: string): string { const end = readIdentifier(rawName, 0) diff --git a/packages/opencode/src/altimate/tools/dbt-cte-profile.ts b/packages/opencode/src/altimate/tools/dbt-cte-profile.ts index bb005b800d..685f3b5b97 100644 --- a/packages/opencode/src/altimate/tools/dbt-cte-profile.ts +++ b/packages/opencode/src/altimate/tools/dbt-cte-profile.ts @@ -7,7 +7,7 @@ import { projectRoot } from "../native/connections/registry" import { isQueryTimeoutError } from "@altimateai/drivers/types" import { executeQuery } from "../native/query-execution" import { dbtAdapterFor } from "./dbt-adapter-access" -import { buildCountQuery, detectCtes, extractRowCount, stripColumnList, unclosedJinjaBlock, type CteInfo } from "./cte-detect" +import { buildCountQuery, detectCtes, extractRowCount, stripColumnList, unclosedJinjaBlock, unreadCteAfter, type CteInfo } from "./cte-detect" /** Folders a model search never enters. */ const SKIPPED_DIRS = new Set(["target", "dbt_packages", "dbt_modules", "node_modules", "logs", "venv", "site-packages"]) @@ -136,7 +136,8 @@ export const DbtCteProfileTool = Tool.define("dbt_cte_profile", { if (!root) return failure("no dbt project", "No dbt project here: run from a dbt project folder.") const located = modelFile(root, args.model) if (!located.file) return failure("model not found", located.error ?? `no model named ${args.model}`) - const size = fs.statSync(located.file).size + const modelPath = located.file + const size = fs.statSync(modelPath).size if (size > MAX_MODEL_BYTES) return failure("model too large", `${args.model} is ${size} bytes; the largest file read is ${MAX_MODEL_BYTES}.`) const text = fs.readFileSync(located.file, "utf-8") const ctes = detectCtes(text) @@ -157,12 +158,14 @@ export const DbtCteProfileTool = Tool.define("dbt_cte_profile", { clause: c.withClauseStart, position, })) - const output = list.length - ? [`${list.length} CTE${list.length === 1 ? "" : "s"} in ${file}:`, ...list.map((c) => `${c.position + 1}. ${c.name} (line ${c.line})`)].join("\n") - : `No CTEs in ${file}.` + const stop = unreadCteAfter(text, ctes) + const note = stop === undefined ? "" : `\nThe list is incomplete: a CTE after line ${lineAt(stop)} could not be read (its name is built by a Jinja expression or loop).` + const output = (list.length + ? [`${list.length} CTE${list.length === 1 ? "" : "s"} in ${file}:`, ...list.map((c) => `position ${c.position}: ${c.name} (line ${c.line})`)].join("\n") + : `No CTEs in ${file}.`) + note return { title: `dbt_cte_profile: ${list.length} CTE${list.length === 1 ? "" : "s"} in ${args.model}`, - metadata: { success: true, model: args.model, file, ctes: list }, + metadata: { success: true, model: args.model, file, ctes: list, ...(stop === undefined ? {} : { incomplete: true }) }, output, } } @@ -213,7 +216,7 @@ export const DbtCteProfileTool = Tool.define("dbt_cte_profile", { try { // The adapter has no cancel of its own: the run's timeout, budget and abort end the wait, not the query. const result: { data?: Record[] } = await executeQuery({ type: "dbt" }, {}, { signal: ctx.abort, sessionID: ctx.sessionID }, () => - adapter.immediatelyExecuteSQLWithLimit(query, `cte_profiler_${name}`, 1), + adapter.immediatelyExecuteSQLWithLimit(query, path.basename(modelPath, ".sql"), 1), ) const queryMs = Math.round(performance.now() - started) const rows = extractRowCount(result?.data) diff --git a/packages/opencode/test/altimate/dbt-cte-profile.test.ts b/packages/opencode/test/altimate/dbt-cte-profile.test.ts index 60f33384cf..d1dabe2764 100644 --- a/packages/opencode/test/altimate/dbt-cte-profile.test.ts +++ b/packages/opencode/test/altimate/dbt-cte-profile.test.ts @@ -128,9 +128,32 @@ describe("dbt_cte_profile list", () => { { name: "final", line: 12, index: 2, clause: expect.any(Number), position: 2 }, ]) expect(result.output).toContain("3 CTEs") + // The numbers shown are the ones `position` takes: it counts from 0. + expect(result.output).toContain("position 0: orders (line 4)") + expect(result.output).toContain("position 2: final (line 12)") expect(queries).toEqual([]) }) + test("a CTE whose name a Jinja expression builds ends the list, and the list says so", async () => { + await writeFile( + path.join(project, "models", "marts", "looped.sql"), + "with base as (select 1 as a),\n{% for t in ['x'] %}\n{{ t }}_c as (select 1 as a),\n{% endfor %}\nfinal as (select 1 as a) select * from final\n", + ) + const tool = await initTool(DbtCteProfileTool) + const result = await tool.execute({ model: "looped" }, ctx) + expect(result.metadata.ctes.map((c: { name: string }) => c.name)).toEqual(["base"]) + expect(result.metadata.incomplete).toBe(true) + expect(result.output).toContain("The list is incomplete: a CTE after line 1") + const whole = await tool.execute({ model: "customers" }, ctx) + expect(whole.metadata.incomplete).toBeUndefined() + }) + + test("the query runs against the model's own name, so its incremental context is kept", async () => { + const tool = await initTool(DbtCteProfileTool) + await tool.execute({ model: "customers", cte: "orders" }, ctx) + expect(queries[0].name).toBe("customers") + }) + test("a model without CTEs lists none", async () => { const tool = await initTool(DbtCteProfileTool) const result = await tool.execute({ model: "plain" }, ctx) @@ -355,7 +378,7 @@ describe("dbt_cte_profile run", () => { expect(result.metadata.rows).toBe(42) expect(typeof result.metadata.queryMs).toBe("number") expect(queries).toHaveLength(1) - expect(queries[0].name).toBe('cte_profiler_"Big Spenders"') + expect(queries[0].name).toBe("customers") expect(queries[0].limit).toBe(1) expect(queries[0].sql).toContain("{{ config(materialized='table') }}") expect(queries[0].sql).toContain("orders as (") From 493307acbeef63922b768977ff795639b5d5de51 Mon Sep 17 00:00:00 2001 From: Harish Kumar Date: Tue, 17 Nov 2026 08:04:22 +0530 Subject: [PATCH 17/26] fix: dbt_cte_profile flags a WITH clause whose first CTE is Jinja-built, reports the right line, and says so when a lookup misses Co-Authored-By: Claude Sonnet 5.5 --- .../opencode/src/altimate/tools/cte-detect.ts | 22 +++++++++++++------ .../src/altimate/tools/dbt-cte-profile.ts | 8 ++++--- .../test/altimate/dbt-cte-profile.test.ts | 12 ++++++++++ 3 files changed, 32 insertions(+), 10 deletions(-) diff --git a/packages/opencode/src/altimate/tools/cte-detect.ts b/packages/opencode/src/altimate/tools/cte-detect.ts index 1b9658ac92..3b41745afe 100644 --- a/packages/opencode/src/altimate/tools/cte-detect.ts +++ b/packages/opencode/src/altimate/tools/cte-detect.ts @@ -302,16 +302,24 @@ export function detectCtes(text: string): CteInfo[] { } /** - * Line-independent offset of a comma that follows the last CTE read of a WITH clause: another CTE comes - * after it but could not be read (its name is built by a Jinja expression or loop), so the list is short. + * Offset where a CTE that could not be read begins or is announced, or undefined when every CTE was read: the + * comma after the last CTE read of a WITH clause that goes on, or the start of a clause where none was read + * (its first name is built by a Jinja expression or loop). The list is short from there on. */ export function unreadCteAfter(text: string, ctes: CteInfo[]): number | undefined { - for (const c of ctes) { - if (ctes.some((other) => other.withClauseStart === c.withClauseStart && other.index === c.index + 1)) continue - const next = skipTrivia(text, c.queryEnd + 1, true) - if (text[next] === ",") return next + const found: number[] = [] + for (const withPos of findTopLevelWithKeywords(text)) { + const own = ctes.filter((c) => c.withClauseStart === withPos) + if (own.length === 0) { + const first = skipTrivia(text, withPos + 4, true) + if (text[first] === "{" && text[first + 1] === "{") found.push(first) + continue + } + const last = own[own.length - 1] + const next = skipTrivia(text, last.queryEnd + 1, true) + if (text[next] === ",") found.push(next) } - return undefined + return found.length ? Math.min(...found) : undefined } /** Drop a trailing column list: `my_cte (id, name)` -> `my_cte`. */ diff --git a/packages/opencode/src/altimate/tools/dbt-cte-profile.ts b/packages/opencode/src/altimate/tools/dbt-cte-profile.ts index 685f3b5b97..8300ca23a1 100644 --- a/packages/opencode/src/altimate/tools/dbt-cte-profile.ts +++ b/packages/opencode/src/altimate/tools/dbt-cte-profile.ts @@ -159,7 +159,7 @@ export const DbtCteProfileTool = Tool.define("dbt_cte_profile", { position, })) const stop = unreadCteAfter(text, ctes) - const note = stop === undefined ? "" : `\nThe list is incomplete: a CTE after line ${lineAt(stop)} could not be read (its name is built by a Jinja expression or loop).` + const note = stop === undefined ? "" : `\nThe list is incomplete: a CTE after line ${text.substring(0, stop).split("\n").length} could not be read (its name is built by a Jinja expression or loop).` const output = (list.length ? [`${list.length} CTE${list.length === 1 ? "" : "s"} in ${file}:`, ...list.map((c) => `position ${c.position}: ${c.name} (line ${c.line})`)].join("\n") : `No CTEs in ${file}.`) + note @@ -170,11 +170,13 @@ export const DbtCteProfileTool = Tool.define("dbt_cte_profile", { } } + const unread = unreadCteAfter(text, ctes) !== undefined + const incompleteNote = unread ? " The list is incomplete: a CTE built by a Jinja expression or loop could not be read." : "" const names = ctes.map((c) => stripColumnList(c.name)).join(", ") let target: CteInfo | undefined if (args.position !== undefined) { target = ctes[args.position] - if (!target) return failure("CTE not found", `${args.model} has ${ctes.length} CTEs: there is none at position ${args.position}.`, { model: args.model }) + if (!target) return failure("CTE not found", `${args.model} has ${ctes.length} CTEs: there is none at position ${args.position}.${incompleteNote}`, { model: args.model }) // The name as typed wins here too: another CTE spelled exactly so is the one meant, whatever the case of this one. const typed = args.cte === undefined ? undefined : bareName(args.cte) const spelledSo = typed === undefined ? false : ctes.some((c) => bareName(c.name) === typed) @@ -186,7 +188,7 @@ export const DbtCteProfileTool = Tool.define("dbt_cte_profile", { const exact = ctes.filter((c) => bareName(c.name) === wanted) const matches = exact.length > 0 ? exact : ctes.filter((c) => bareName(c.name).toLowerCase() === wanted.toLowerCase()) if (matches.length === 0) - return failure("CTE not found", `${args.model} has no CTE named ${args.cte}${names ? `; it has: ${names}` : ""}.`, { model: args.model }) + return failure("CTE not found", `${args.model} has no CTE named ${args.cte}${names ? `; it has: ${names}` : ""}.${incompleteNote}`, { model: args.model }) if (matches.length > 1) { const at = matches.map((c) => ctes.indexOf(c)).join(", ") return failure("CTE ambiguous", `${args.model} has ${matches.length} CTEs that match ${args.cte} (positions ${at}): pass position.`, { model: args.model }) diff --git a/packages/opencode/test/altimate/dbt-cte-profile.test.ts b/packages/opencode/test/altimate/dbt-cte-profile.test.ts index d1dabe2764..7bf5b65c0f 100644 --- a/packages/opencode/test/altimate/dbt-cte-profile.test.ts +++ b/packages/opencode/test/altimate/dbt-cte-profile.test.ts @@ -144,6 +144,18 @@ describe("dbt_cte_profile list", () => { expect(result.metadata.ctes.map((c: { name: string }) => c.name)).toEqual(["base"]) expect(result.metadata.incomplete).toBe(true) expect(result.output).toContain("The list is incomplete: a CTE after line 1") + const absent = await tool.execute({ model: "looped", cte: "final" }, ctx) + expect(absent.metadata.success).toBe(false) + expect(absent.output).toContain("The list is incomplete") + await writeFile( + path.join(project, "models", "marts", "firstloop.sql"), + "{% if true %}\nwith a as (select 1 as x), {{ y }}_c as (select 1 as x) select * from a\n{% else %}\nwith b as (select 2 as x) select * from b\n{% endif %}\n", + ) + const multi = await tool.execute({ model: "firstloop" }, ctx) + expect(multi.output).toContain("after line 2 could not be read") + await writeFile(path.join(project, "models", "marts", "loopfirst.sql"), "with {{ x }}_c as (select 1 as a), final as (select 1 as a) select * from final\n") + const first = await tool.execute({ model: "loopfirst" }, ctx) + expect(first.metadata.incomplete).toBe(true) const whole = await tool.execute({ model: "customers" }, ctx) expect(whole.metadata.incomplete).toBeUndefined() }) From e591307540bc4977e4cfe9758f6482c0e811a5d3 Mon Sep 17 00:00:00 2001 From: Harish Kumar Date: Tue, 17 Nov 2026 08:06:55 +0530 Subject: [PATCH 18/26] fix: dbt_cte_profile's incomplete-list note covers a prefixed or recursive Jinja first name and leaves START WITH alone Co-Authored-By: Claude Sonnet 5.5 --- .../opencode/src/altimate/tools/cte-detect.ts | 10 +++++++--- .../src/altimate/tools/dbt-cte-profile.ts | 2 +- .../test/altimate/dbt-cte-profile.test.ts | 19 +++++++++++++++++-- 3 files changed, 25 insertions(+), 6 deletions(-) diff --git a/packages/opencode/src/altimate/tools/cte-detect.ts b/packages/opencode/src/altimate/tools/cte-detect.ts index 3b41745afe..816aa6405f 100644 --- a/packages/opencode/src/altimate/tools/cte-detect.ts +++ b/packages/opencode/src/altimate/tools/cte-detect.ts @@ -302,7 +302,7 @@ export function detectCtes(text: string): CteInfo[] { } /** - * Offset where a CTE that could not be read begins or is announced, or undefined when every CTE was read: the + * Offset from which the CTEs could not be read, or undefined when every CTE was read: the * comma after the last CTE read of a WITH clause that goes on, or the start of a clause where none was read * (its first name is built by a Jinja expression or loop). The list is short from there on. */ @@ -311,8 +311,12 @@ export function unreadCteAfter(text: string, ctes: CteInfo[]): number | undefine for (const withPos of findTopLevelWithKeywords(text)) { const own = ctes.filter((c) => c.withClauseStart === withPos) if (own.length === 0) { - const first = skipTrivia(text, withPos + 4, true) - if (text[first] === "{" && text[first + 1] === "{") found.push(first) + // `START WITH` (a hierarchical query) opens no CTE list. + if (/\bstart\s*$/i.test(text.substring(Math.max(0, withPos - 24), withPos))) continue + let first = skipTrivia(text, withPos + 4, true) + if (/recursive\b/iy.test(text.substring(first, first + 10))) first = skipTrivia(text, first + "recursive".length, true) + const unreadName = /^[\p{L}\p{N}_$]*\{\{/u.exec(text.substring(first, first + 130)) + if (unreadName) found.push(first) continue } const last = own[own.length - 1] diff --git a/packages/opencode/src/altimate/tools/dbt-cte-profile.ts b/packages/opencode/src/altimate/tools/dbt-cte-profile.ts index 8300ca23a1..102e33dd00 100644 --- a/packages/opencode/src/altimate/tools/dbt-cte-profile.ts +++ b/packages/opencode/src/altimate/tools/dbt-cte-profile.ts @@ -159,7 +159,7 @@ export const DbtCteProfileTool = Tool.define("dbt_cte_profile", { position, })) const stop = unreadCteAfter(text, ctes) - const note = stop === undefined ? "" : `\nThe list is incomplete: a CTE after line ${text.substring(0, stop).split("\n").length} could not be read (its name is built by a Jinja expression or loop).` + const note = stop === undefined ? "" : `\nThe list is incomplete: the CTEs from line ${text.substring(0, stop).split("\n").length} on could not be read (its name is built by a Jinja expression or loop).` const output = (list.length ? [`${list.length} CTE${list.length === 1 ? "" : "s"} in ${file}:`, ...list.map((c) => `position ${c.position}: ${c.name} (line ${c.line})`)].join("\n") : `No CTEs in ${file}.`) + note diff --git a/packages/opencode/test/altimate/dbt-cte-profile.test.ts b/packages/opencode/test/altimate/dbt-cte-profile.test.ts index 7bf5b65c0f..78b41d3093 100644 --- a/packages/opencode/test/altimate/dbt-cte-profile.test.ts +++ b/packages/opencode/test/altimate/dbt-cte-profile.test.ts @@ -143,7 +143,7 @@ describe("dbt_cte_profile list", () => { const result = await tool.execute({ model: "looped" }, ctx) expect(result.metadata.ctes.map((c: { name: string }) => c.name)).toEqual(["base"]) expect(result.metadata.incomplete).toBe(true) - expect(result.output).toContain("The list is incomplete: a CTE after line 1") + expect(result.output).toContain("The list is incomplete: the CTEs from line 1 on could not be read") const absent = await tool.execute({ model: "looped", cte: "final" }, ctx) expect(absent.metadata.success).toBe(false) expect(absent.output).toContain("The list is incomplete") @@ -152,10 +152,25 @@ describe("dbt_cte_profile list", () => { "{% if true %}\nwith a as (select 1 as x), {{ y }}_c as (select 1 as x) select * from a\n{% else %}\nwith b as (select 2 as x) select * from b\n{% endif %}\n", ) const multi = await tool.execute({ model: "firstloop" }, ctx) - expect(multi.output).toContain("after line 2 could not be read") + expect(multi.output).toContain("the CTEs from line 2 on could not be read") await writeFile(path.join(project, "models", "marts", "loopfirst.sql"), "with {{ x }}_c as (select 1 as a), final as (select 1 as a) select * from final\n") const first = await tool.execute({ model: "loopfirst" }, ctx) expect(first.metadata.incomplete).toBe(true) + expect(first.metadata.ctes).toEqual([]) + expect(first.output).toContain("No CTEs in") + expect(first.output).toContain("the CTEs from line 1 on could not be read") + for (const [model, sql] of [ + ["prefixed", "with recursive stg_{{ var('s') }} as (select 1 as a) select * from stg_x\n"], + ["hierarchy", "select * from emp start with {{ root_filter() }} connect by prior id = parent_id\n"], + ]) { + await writeFile(path.join(project, "models", "marts", `${model}.sql`), sql) + } + expect((await tool.execute({ model: "prefixed" }, ctx)).metadata.incomplete).toBe(true) + const hierarchy = await tool.execute({ model: "hierarchy" }, ctx) + expect(hierarchy.metadata.incomplete).toBeUndefined() + expect(hierarchy.output).toBe("No CTEs in " + path.join("models", "marts", "hierarchy.sql") + ".") + const byPosition = await tool.execute({ model: "looped", position: 5 }, ctx) + expect(byPosition.output).toContain("The list is incomplete") const whole = await tool.execute({ model: "customers" }, ctx) expect(whole.metadata.incomplete).toBeUndefined() }) From 921ac71e16fb6252d0e1b310c53d1ea358f1dff9 Mon Sep 17 00:00:00 2001 From: Harish Kumar Date: Tue, 17 Nov 2026 11:43:48 +0530 Subject: [PATCH 19/26] fix: dbt_cte_profile refuses a CTE chain that reads is_incremental() or this, since the count query runs outside the model's context Co-Authored-By: Claude Sonnet 5.5 --- .../src/altimate/tools/dbt-cte-profile.ts | 3 +++ .../test/altimate/dbt-cte-profile.test.ts | 17 ++++++++++++++++- 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/packages/opencode/src/altimate/tools/dbt-cte-profile.ts b/packages/opencode/src/altimate/tools/dbt-cte-profile.ts index 102e33dd00..230e80ed5d 100644 --- a/packages/opencode/src/altimate/tools/dbt-cte-profile.ts +++ b/packages/opencode/src/altimate/tools/dbt-cte-profile.ts @@ -210,6 +210,9 @@ export const DbtCteProfileTool = Tool.define("dbt_cte_profile", { const block = unclosedJinjaBlock(query) if (block) return failure("no query", `${name} is inside a Jinja {% ${block} %} block of ${args.model}, which cannot be cut off at that CTE and stay valid: compile the model and profile the compiled SQL, or move the WITH out of the block.`, { model: args.model, cte: name }) + // The adapter runs the query as an inline node, not as the model, so is_incremental() and this would resolve for that node instead. + if (/\bis_incremental\s*\(|\{\{-?[^}]*\bthis\b/.test(query)) return failure("not run", `${name} depends on is_incremental() or this in ${args.model}. A count query runs outside the model, so those would resolve differently and the count would not match what the model reads. Compile the model and profile the compiled SQL instead.`, { model: args.model, cte: name }) + const access = await dbtAdapterFor(ctx) if (!access.adapter) return failure("not run", access.reason ?? "the dbt adapter is not available", { model: args.model, cte: name, refusal: access.refusal }) const adapter = access.adapter diff --git a/packages/opencode/test/altimate/dbt-cte-profile.test.ts b/packages/opencode/test/altimate/dbt-cte-profile.test.ts index 78b41d3093..8588ebe0b8 100644 --- a/packages/opencode/test/altimate/dbt-cte-profile.test.ts +++ b/packages/opencode/test/altimate/dbt-cte-profile.test.ts @@ -175,12 +175,27 @@ describe("dbt_cte_profile list", () => { expect(whole.metadata.incomplete).toBeUndefined() }) - test("the query runs against the model's own name, so its incremental context is kept", async () => { + test("the query runs against the model's own name", async () => { const tool = await initTool(DbtCteProfileTool) await tool.execute({ model: "customers", cte: "orders" }, ctx) expect(queries[0].name).toBe("customers") }) + test("a CTE chain that reads is_incremental() or this is not run, since the count query would not see the model's context", async () => { + const tool = await initTool(DbtCteProfileTool) + for (const [model, sql] of [ + ["inc_filter", "with a as (select * from {{ ref('x') }} {% if is_incremental() %} where id > 1 {% endif %}), b as (select * from a) select * from b"], + ["inc_this", "with a as (select max(id) m from {{ this }}), b as (select * from a) select * from b"], + ] as const) { + await writeFile(path.join(project, "models", "marts", `${model}.sql`), sql) + const before = queries.length + const result = await tool.execute({ model, cte: "b" }, ctx) + expect(result.metadata.success).toBe(false) + expect(result.output).toContain("is_incremental() or this") + expect(queries.length).toBe(before) + } + }) + test("a model without CTEs lists none", async () => { const tool = await initTool(DbtCteProfileTool) const result = await tool.execute({ model: "plain" }, ctx) From 8794c16ffcc9674e045784e6f3d23c062fb8d1fa Mon Sep 17 00:00:00 2001 From: Harish Kumar Date: Tue, 17 Nov 2026 11:49:36 +0530 Subject: [PATCH 20/26] fix: dbt_cte_profile's is_incremental()/this refusal reads only Jinja tags, so comments, strings and ref names no longer trip it, and covers set blocks Co-Authored-By: Claude Sonnet 5.5 --- .../opencode/src/altimate/tools/dbt-cte-profile.ts | 8 +++++++- .../opencode/test/altimate/dbt-cte-profile.test.ts | 11 +++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/packages/opencode/src/altimate/tools/dbt-cte-profile.ts b/packages/opencode/src/altimate/tools/dbt-cte-profile.ts index 230e80ed5d..586ce014f7 100644 --- a/packages/opencode/src/altimate/tools/dbt-cte-profile.ts +++ b/packages/opencode/src/altimate/tools/dbt-cte-profile.ts @@ -112,6 +112,12 @@ async function projectDir(): Promise { return getDbtProjectRoot() } +// True when a Jinja tag (comments and quoted strings inside it left out) calls is_incremental() or names this. +function readsModelContext(sql: string): boolean { + const tags = sql.replace(/\{#[\s\S]*?#\}/g, "").match(/\{\{[\s\S]*?\}\}|\{%[\s\S]*?%\}/g) ?? [] + return tags.some((tag) => /\bis_incremental\s*\(|\bthis\b/.test(tag.replace(/'(?:[^'\\]|\\.)*'|"(?:[^"\\]|\\.)*"/g, "''"))) +} + function failure(title: string, output: string, extra: Record = {}) { return { title: `dbt_cte_profile: ${title}`, metadata: { success: false, error: output, ...extra }, output } } @@ -211,7 +217,7 @@ export const DbtCteProfileTool = Tool.define("dbt_cte_profile", { if (block) return failure("no query", `${name} is inside a Jinja {% ${block} %} block of ${args.model}, which cannot be cut off at that CTE and stay valid: compile the model and profile the compiled SQL, or move the WITH out of the block.`, { model: args.model, cte: name }) // The adapter runs the query as an inline node, not as the model, so is_incremental() and this would resolve for that node instead. - if (/\bis_incremental\s*\(|\{\{-?[^}]*\bthis\b/.test(query)) return failure("not run", `${name} depends on is_incremental() or this in ${args.model}. A count query runs outside the model, so those would resolve differently and the count would not match what the model reads. Compile the model and profile the compiled SQL instead.`, { model: args.model, cte: name }) + if (readsModelContext(query)) return failure("not run", `${name} depends on is_incremental() or this in ${args.model}. A count query runs outside the model, so those would resolve differently and the count would not match what the model reads. Compile the model and profile the compiled SQL instead.`, { model: args.model, cte: name }) const access = await dbtAdapterFor(ctx) if (!access.adapter) return failure("not run", access.reason ?? "the dbt adapter is not available", { model: args.model, cte: name, refusal: access.refusal }) diff --git a/packages/opencode/test/altimate/dbt-cte-profile.test.ts b/packages/opencode/test/altimate/dbt-cte-profile.test.ts index 8588ebe0b8..b2f6d491c0 100644 --- a/packages/opencode/test/altimate/dbt-cte-profile.test.ts +++ b/packages/opencode/test/altimate/dbt-cte-profile.test.ts @@ -186,6 +186,7 @@ describe("dbt_cte_profile list", () => { for (const [model, sql] of [ ["inc_filter", "with a as (select * from {{ ref('x') }} {% if is_incremental() %} where id > 1 {% endif %}), b as (select * from a) select * from b"], ["inc_this", "with a as (select max(id) m from {{ this }}), b as (select * from a) select * from b"], + ["inc_set", "{% set rel = this %} with a as (select * from {{ rel }}), b as (select * from a) select * from b"], ] as const) { await writeFile(path.join(project, "models", "marts", `${model}.sql`), sql) const before = queries.length @@ -196,6 +197,16 @@ describe("dbt_cte_profile list", () => { } }) + test("is_incremental() or this in a comment, a string or a ref name does not stop the run", async () => { + const tool = await initTool(DbtCteProfileTool) + await writeFile( + path.join(project, "models", "marts", "quiet.sql"), + "{# is_incremental() is unused, {{ this }} #}\n-- is_incremental()\nwith a as (select 'is_incremental()' as label from {{ ref('this_model') }}), b as (select * from a) select * from b", + ) + const result = await tool.execute({ model: "quiet", cte: "b" }, ctx) + expect(result.metadata.success).toBe(true) + }) + test("a model without CTEs lists none", async () => { const tool = await initTool(DbtCteProfileTool) const result = await tool.execute({ model: "plain" }, ctx) From aaff4148345d9e6735a119d27ea540a3d2929a9d Mon Sep 17 00:00:00 2001 From: Harish Kumar Date: Tue, 17 Nov 2026 13:21:13 +0530 Subject: [PATCH 21/26] fix: never echo a send failure's text, so a trimmed key can't leak Bun trims a header value before quoting it in its validation error, so redacting exact copies of the stored key missed keys with a line break and trailing whitespace. A send failure now reports only a fixed reason for a refused header, or the error's name and code otherwise. Also: a missing ${env:...} variable no longer quotes its name from the credentials file; a credentials directory that can't be searched is reported as unreadable, not as no credentials; the adapter deadline says "wasn't ready" since it covers the config read too. Regression tests go through the real fetch for CR/LF and trailing-whitespace keys, via apiRequest and listDatamates, and check stubs still validate input. Co-Authored-By: Claude Opus 5.5 --- packages/opencode/src/altimate/api/client.ts | 40 ++++-- .../src/altimate/tools/dbt-adapter-access.ts | 2 +- .../test/altimate/dbt-adapter-access.test.ts | 121 ++++++++++++++---- 3 files changed, 126 insertions(+), 37 deletions(-) diff --git a/packages/opencode/src/altimate/api/client.ts b/packages/opencode/src/altimate/api/client.ts index 25a78d8bd0..a7a668183e 100644 --- a/packages/opencode/src/altimate/api/client.ts +++ b/packages/opencode/src/altimate/api/client.ts @@ -1,6 +1,6 @@ import z from "zod" import path from "path" -import { rm } from "node:fs/promises" +import { rm, stat } from "node:fs/promises" import { Global } from "../../global" import { Filesystem } from "../../util/filesystem" import { currentCredentials } from "./credentials-context" @@ -86,11 +86,14 @@ export namespace AltimateApi { return Filesystem.exists(credentialsPath()) } + /** A `${env:NAME}` in the credentials file names a variable that isn't set. */ + class MissingEnvVar extends Error {} + function resolveEnvVars(obj: unknown): unknown { if (typeof obj === "string") { return obj.replace(/\$\{env:([^}]+)\}/g, (_, envVar) => { const value = process.env[envVar] - if (value === undefined) throw new Error(`Environment variable ${envVar} not found`) + if (value === undefined) throw new MissingEnvVar(`Environment variable ${envVar} not found`) return value }) } @@ -324,9 +327,18 @@ export namespace AltimateApi { timeoutMs: 15_000, } - /** `text` with every occurrence of `secret` replaced, so an error can be shown without the key. */ - function redact(text: string, secret: string): string { - return secret ? text.split(secret).join("[redacted]") : text + /** + * Why a request couldn't be sent, never quoting the error's text. Bun refuses a header value with a + * control character as a TypeError that quotes the value after trimming it, so no redaction of the + * stored key can be relied on: only the error's name and code are shown. + */ + function sendFailure(e: unknown): string { + if (e instanceof TypeError) + return "the request was refused before it was sent (TypeError): the stored API key or tenant name may not be a valid header value." + const word = (v: unknown) => (typeof v === "string" && /^[A-Za-z][A-Za-z0-9_]{0,63}$/.test(v) ? v : undefined) + const name = (e instanceof Error && word(e.name)) || "Error" + const code = word((e as { code?: unknown } | null)?.code) + return code ? `${name} (${code}).` : `${name}.` } async function request( @@ -346,8 +358,7 @@ export namespace AltimateApi { // that raced a refused header leaves the header error, so match the reason, not the flag. if (e === controller.signal.reason || e === signal?.reason) throw e // Others can quote a header: Bun's check of a header value with CR/LF quotes `Bearer `. - const message = redact(e instanceof Error ? e.message : String(e), creds.altimateApiKey) - throw new Error(`API ${method} ${endpoint} could not be sent: ${message}`) + throw new Error(`API ${method} ${endpoint} could not be sent: ${sendFailure(e)}`) } } // altimate_change start — upstream_fix: bound every API request. Without a @@ -394,7 +405,9 @@ export namespace AltimateApi { if (fields.length === 0 || fields.includes("")) return "the file does not hold the credentials object." return `${fields.join(", ")} ${fields.length === 1 ? "is" : "are"} missing or not valid.` } - // A file system error or a missing ${env:NAME} variable: names a path or a variable, no value. + // The variable's name is the file's text, which can be a key written in its place. + if (e instanceof MissingEnvVar) return "an ${env:…} variable it refers to is not set." + // A file system error names a path, not a value. return e instanceof Error ? e.message : String(e) } @@ -403,7 +416,7 @@ export namespace AltimateApi { * returns the parsed JSON body. Endpoint functions in other files go through this. Fails with a * message naming the credentials file when none are stored or they can't be read. `signal` (a tool * call's `ctx.abort`) cancels the request as well as the 15 s timeout. No error it raises quotes the - * stored key. + * stored key or the credentials file's text. */ export async function apiRequest( method: string, @@ -417,7 +430,14 @@ export namespace AltimateApi { creds = await getCredentials() } catch (e) { const p = credentialsPath() - if (!(await Filesystem.exists(p))) + // Only a file that isn't there is "not stored": a directory without search permission hides it too. + const unseen = await stat(p).then( + () => undefined, + (err: NodeJS.ErrnoException) => err, + ) + if (unseen && unseen.code !== "ENOENT" && unseen.code !== "ENOTDIR") + throw new Error(`The Altimate credentials in ${p} can't be read (${unseen.code ?? unseen.name}).`) + if (unseen) throw new Error( `No Altimate credentials are stored: sign in to Altimate in altimate-code first (they are saved to ${p}).`, ) diff --git a/packages/opencode/src/altimate/tools/dbt-adapter-access.ts b/packages/opencode/src/altimate/tools/dbt-adapter-access.ts index 2fc2aba423..d51d2b4d2b 100644 --- a/packages/opencode/src/altimate/tools/dbt-adapter-access.ts +++ b/packages/opencode/src/altimate/tools/dbt-adapter-access.ts @@ -42,7 +42,7 @@ export async function dbtAdapterFor(ctx: Pick reject(new Error(`the dbt adapter didn't start within ${seconds} s`)), + () => reject(new Error(`the dbt adapter wasn't ready within ${seconds} s`)), dbtAdapterAccessInternals.adapterStartTimeoutMs, ) }) diff --git a/packages/opencode/test/altimate/dbt-adapter-access.test.ts b/packages/opencode/test/altimate/dbt-adapter-access.test.ts index ddcf0ea183..6e3d7958c6 100644 --- a/packages/opencode/test/altimate/dbt-adapter-access.test.ts +++ b/packages/opencode/test/altimate/dbt-adapter-access.test.ts @@ -2,7 +2,7 @@ // one gate the new dbt-adapter-backed tools go through, and the API helper for new endpoint files. The // dbt adapter is replaced through `dbtAdapterInternals`, so these tests need no Python or dbt. import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, test } from "bun:test" -import { mkdtemp, rm, writeFile } from "node:fs/promises" +import { chmod, mkdtemp, rm, writeFile } from "node:fs/promises" import { realpathSync } from "node:fs" import os from "node:os" import path from "node:path" @@ -93,6 +93,25 @@ describe("feature tools in the allow-list", () => { expect(resultFailed(result)).toBe(true) } }) + + test("each placeholder still checks its arguments before it refuses", async () => { + for (const id of FEATURE_TOOLS) { + const tool = await initTool(EXPOSED[id] as any) + // A string where an object is expected fails every schema, whatever its required fields. + const failed = await Promise.resolve() + .then(() => tool.execute("not an object" as any, ctx)) + .then( + (result) => ({ result }), + (error: unknown) => ({ error }), + ) + if ("result" in failed) { + expect(failed.result.title).not.toBe(`${id}: not implemented`) + expect(resultFailed(failed.result)).toBe(true) + } else { + expect(String(failed.error)).toContain("invalid arguments") + } + } + }) }) describe("dbtAdapterFor", () => { @@ -157,7 +176,7 @@ describe("dbtAdapterFor", () => { dbtAdapterAccessInternals.adapterStartTimeoutMs = 20 try { const access = await dbtAdapterFor(ctx) - expect(access).toEqual({ refusal: "not_started", reason: "the dbt adapter didn't start within 0.02 s" }) + expect(access).toEqual({ refusal: "not_started", reason: "the dbt adapter wasn't ready within 0.02 s" }) } finally { dbtAdapterAccessInternals.adapterStartTimeoutMs = saved } @@ -224,7 +243,7 @@ describe("dbtAdapterFor", () => { dbtAdapterAccessInternals.adapterStartTimeoutMs = 20 try { const access = await Instance.provide({ directory: project, fn: () => dbtAdapterFor(ctx) }) - expect(access).toEqual({ refusal: "not_started", reason: "the dbt adapter didn't start within 0.02 s" }) + expect(access).toEqual({ refusal: "not_started", reason: "the dbt adapter wasn't ready within 0.02 s" }) expect(created).toBe(0) } finally { release() @@ -427,17 +446,53 @@ describe("AltimateApi.apiRequest", () => { expect(error.message).not.toContain("sk_SECRET123") }) - test("never shows the stored key when sending the request fails", async () => { - const key = "sk_SECRET123\r\nX: y" + const HEADER_REFUSED = + "could not be sent: the request was refused before it was sent (TypeError): the stored API key or tenant name may not be a valid header value." + + /** Every substring of `key` long enough to identify it, so a trimmed or split copy is caught too. */ + function expectNoPartOf(text: string, key: string) { + for (const part of key.split(/\s+/).filter((p) => p.length >= 4)) expect(text).not.toContain(part) + expect(text).not.toContain("Bearer") + } + + // The real fetch: Bun refuses a header value with CR/LF, quoting it after trimming its whitespace. + for (const key of [ + "sk_SECRET123\r\nX: y", + "sk_SECRET123ABCDEFGH\nPART2\n", + "sk_SECRET123ABCDEFGH\r\nX \t", + " sk_SECRET123ABCDEFGH\nPART2 ", + ]) { + test(`never shows any part of a key that isn't a valid header value: ${JSON.stringify(key)}`, async () => { + await Bun.write( + path.join(home, ".altimate", "altimate.json"), + JSON.stringify({ altimateUrl: "http://127.0.0.1:9", altimateInstanceName: "acme", altimateApiKey: key }), + ) + const viaRequest = await AltimateApi.apiRequest("GET", "/dbt/v1/x").catch((e: Error) => e) + expect(viaRequest).toBeInstanceOf(Error) + expect(viaRequest.message).toBe(`API GET /dbt/v1/x ${HEADER_REFUSED}`) + expectNoPartOf(`${viaRequest.message}\n${viaRequest.stack}`, key) + const viaList = (await AltimateApi.listDatamates().catch((e: Error) => e)) as Error + expect(viaList).toBeInstanceOf(Error) + expect(viaList.message).toBe(`API GET /datamates/ ${HEADER_REFUSED}`) + expectNoPartOf(`${viaList.message}\n${viaList.stack}`, key) + }) + } + + test("names a network failure by its code, never by the error's text", async () => { await Bun.write( path.join(home, ".altimate", "altimate.json"), - JSON.stringify({ altimateUrl: "http://127.0.0.1:9", altimateInstanceName: "acme", altimateApiKey: key }), + JSON.stringify({ altimateUrl: "http://127.0.0.1:9", altimateInstanceName: "acme", altimateApiKey: "k1" }), ) - // The real fetch: it rejects a header value with CR/LF, quoting the value. - const error = await AltimateApi.apiRequest("GET", "/dbt/v1/x").catch((e: Error) => e) - expect(error).toBeInstanceOf(Error) - expect(error.message).not.toContain("sk_SECRET123") - expect(error.message).toContain("API GET /dbt/v1/x could not be sent") + const original = globalThis.fetch + globalThis.fetch = (async () => { + throw Object.assign(new Error("Unable to connect: Bearer k1"), { code: "ConnectionRefused" }) + }) as unknown as typeof fetch + try { + const error = await AltimateApi.apiRequest("GET", "/dbt/v1/x").catch((e: Error) => e) + expect(error.message).toBe("API GET /dbt/v1/x could not be sent: Error (ConnectionRefused).") + } finally { + globalThis.fetch = original + } }) test("never shows the stored key when a header refusal races the caller's abort", async () => { @@ -459,33 +514,47 @@ describe("AltimateApi.apiRequest", () => { const error = await AltimateApi.apiRequest("GET", "/dbt/v1/x", undefined, controller.signal).catch( (e: Error) => e, ) - expect(error.message).toBe("API GET /dbt/v1/x could not be sent: bad header: 'Bearer [redacted]'") + expect(error.message).toBe(`API GET /dbt/v1/x ${HEADER_REFUSED}`) } finally { globalThis.fetch = original } }) - test("never shows the stored key from any fetch failure", async () => { + test("never quotes the credentials file when an ${env:…} variable it names isn't set", async () => { + const file = path.join(home, ".altimate", "altimate.json") await Bun.write( - path.join(home, ".altimate", "altimate.json"), + file, JSON.stringify({ - altimateUrl: "https://api.example.test", + altimateUrl: "https://x.test", altimateInstanceName: "acme", - altimateApiKey: "sk_SECRET123", + altimateApiKey: "${env:sk_SECRET123_UNSET}", }), ) - const original = globalThis.fetch - globalThis.fetch = (async (_url: string, init: RequestInit) => { - throw new TypeError(`bad header: '${(init.headers as Record).Authorization}'`) - }) as unknown as typeof fetch - try { - const error = await AltimateApi.apiRequest("GET", "/dbt/v1/x").catch((e: Error) => e) - expect(error.message).toBe("API GET /dbt/v1/x could not be sent: bad header: 'Bearer [redacted]'") - } finally { - globalThis.fetch = original - } + const error = await AltimateApi.apiRequest("GET", "/dbt/v1/x").catch((e: Error) => e) + expect(error.message).toBe( + `The Altimate credentials in ${file} can't be used: an \${env:…} variable it refers to is not set.`, + ) }) + test.skipIf(process.platform === "win32" || process.getuid?.() === 0)( + "says the credentials can't be read, not that none are stored, when their directory can't be searched", + async () => { + const dir = path.join(home, ".altimate") + const file = path.join(dir, "altimate.json") + await Bun.write( + file, + JSON.stringify({ altimateUrl: "https://x.test", altimateInstanceName: "a", altimateApiKey: "k" }), + ) + await chmod(dir, 0o600) + try { + const error = await AltimateApi.apiRequest("GET", "/dbt/v1/x").catch((e: Error) => e) + expect(error.message).toBe(`The Altimate credentials in ${file} can't be read (EACCES).`) + } finally { + await chmod(dir, 0o700) + } + }, + ) + test("a timed-out request says it timed out", async () => { await Bun.write( path.join(home, ".altimate", "altimate.json"), From 3a98dff47e9a501dae418bbe16098ee2fe742550 Mon Sep 17 00:00:00 2001 From: Harish Kumar Date: Tue, 17 Nov 2026 13:29:38 +0530 Subject: [PATCH 22/26] fix: a rejection with no reason is a send failure, not an abort Before either signal aborts, its reason is undefined, so a fetch that rejected with undefined matched it and was rethrown raw. Match a reason only once its signal has aborted. The cleanup test now also covers a timed-out and a cancelled call. Co-Authored-By: Claude Opus 5.5 --- packages/opencode/src/altimate/api/client.ts | 4 +- .../test/altimate/dbt-adapter-access.test.ts | 79 +++++++++++++++---- 2 files changed, 68 insertions(+), 15 deletions(-) diff --git a/packages/opencode/src/altimate/api/client.ts b/packages/opencode/src/altimate/api/client.ts index a7a668183e..123630d58b 100644 --- a/packages/opencode/src/altimate/api/client.ts +++ b/packages/opencode/src/altimate/api/client.ts @@ -356,7 +356,9 @@ export namespace AltimateApi { } catch (e) { // A timeout or the caller's abort rejects with its own reason, which holds no header. An abort // that raced a refused header leaves the header error, so match the reason, not the flag. - if (e === controller.signal.reason || e === signal?.reason) throw e + // A signal that isn't aborted has no reason yet: a rejection with none is a send failure. + if ((controller.signal.aborted && e === controller.signal.reason) || (signal?.aborted && e === signal.reason)) + throw e // Others can quote a header: Bun's check of a header value with CR/LF quotes `Bearer `. throw new Error(`API ${method} ${endpoint} could not be sent: ${sendFailure(e)}`) } diff --git a/packages/opencode/test/altimate/dbt-adapter-access.test.ts b/packages/opencode/test/altimate/dbt-adapter-access.test.ts index 6e3d7958c6..d3a587135e 100644 --- a/packages/opencode/test/altimate/dbt-adapter-access.test.ts +++ b/packages/opencode/test/altimate/dbt-adapter-access.test.ts @@ -276,19 +276,26 @@ describe("dbtAdapterFor", () => { }) test("removes its abort listener and its timer whatever the outcome", async () => { - const controller = new AbortController() - const signal = controller.signal - let listening = 0 - const add = signal.addEventListener.bind(signal) - const remove = signal.removeEventListener.bind(signal) - signal.addEventListener = ((...a: Parameters) => { - if (a[0] === "abort") listening += 1 - return add(...a) - }) as typeof signal.addEventListener - signal.removeEventListener = ((...a: Parameters) => { - if (a[0] === "abort") listening -= 1 - return remove(...a) - }) as typeof signal.removeEventListener + /** A signal that counts the abort listeners still attached to it. */ + const tracked = () => { + const controller = new AbortController() + const signal = controller.signal + const state = { controller, signal, listening: 0 } + const add = signal.addEventListener.bind(signal) + const remove = signal.removeEventListener.bind(signal) + signal.addEventListener = ((...a: Parameters) => { + if (a[0] === "abort") state.listening += 1 + return add(...a) + }) as typeof signal.addEventListener + signal.removeEventListener = ((...a: Parameters) => { + if (a[0] === "abort") state.listening -= 1 + return remove(...a) + }) as typeof signal.removeEventListener + return state + } + const settled = tracked() + const cancelled = tracked() + const signal = settled.signal const timers = new Set() const set = globalThis.setTimeout const clear = globalThis.clearTimeout @@ -308,11 +315,37 @@ describe("dbtAdapterFor", () => { } resetDbtAdapter() expect((await dbtAdapterFor({ ...ctx, abort: signal })).refusal).toBe("not_started") + // Timed out. + dbtAdapterInternals.create = () => new Promise(() => {}) + resetDbtAdapter() + const saved = dbtAdapterAccessInternals.adapterStartTimeoutMs + dbtAdapterAccessInternals.adapterStartTimeoutMs = 20 + try { + expect(await dbtAdapterFor({ ...ctx, abort: signal })).toEqual({ + refusal: "not_started", + reason: "the dbt adapter wasn't ready within 0.02 s", + }) + } finally { + dbtAdapterAccessInternals.adapterStartTimeoutMs = saved + } + // Cancelled while waiting. + let entered = () => {} + const starting = new Promise((resolve) => (entered = resolve)) + dbtAdapterInternals.create = () => { + entered() + return new Promise(() => {}) + } + resetDbtAdapter() + const pending = dbtAdapterFor({ ...ctx, abort: cancelled.signal }) + await starting + cancelled.controller.abort(new Error("cancelled")) + await expect(pending).rejects.toThrow("cancelled") } finally { globalThis.setTimeout = set globalThis.clearTimeout = clear } - expect(listening).toBe(0) + expect(settled.listening).toBe(0) + expect(cancelled.listening).toBe(0) expect(timers.size).toBe(0) }) @@ -478,6 +511,24 @@ describe("AltimateApi.apiRequest", () => { }) } + test("a fetch that rejects with no reason is a send failure, not taken for an abort", async () => { + await Bun.write( + path.join(home, ".altimate", "altimate.json"), + JSON.stringify({ altimateUrl: "http://127.0.0.1:9", altimateInstanceName: "acme", altimateApiKey: "k1" }), + ) + const original = globalThis.fetch + globalThis.fetch = (() => Promise.reject(undefined)) as unknown as typeof fetch + try { + const error = await AltimateApi.apiRequest("GET", "/dbt/v1/x", undefined, new AbortController().signal).catch( + (e: unknown) => e, + ) + expect(error).toBeInstanceOf(Error) + expect((error as Error).message).toBe("API GET /dbt/v1/x could not be sent: Error.") + } finally { + globalThis.fetch = original + } + }) + test("names a network failure by its code, never by the error's text", async () => { await Bun.write( path.join(home, ".altimate", "altimate.json"), From 188d0baa6170f2bf6ec8629be9450ffe2783b7d1 Mon Sep 17 00:00:00 2001 From: Harish Kumar Date: Tue, 17 Nov 2026 13:49:27 +0530 Subject: [PATCH 23/26] fix: dbt_cte_profile's is_incremental()/this check steps over quoted strings and raw blocks, the description says the list can be incomplete, and one helper picks a CTE by name Co-Authored-By: Claude Sonnet 5.5 --- .../src/altimate/tools/dbt-cte-profile.ts | 44 +++++++++++++++---- .../test/altimate/dbt-cte-profile.test.ts | 14 ++++++ 2 files changed, 49 insertions(+), 9 deletions(-) diff --git a/packages/opencode/src/altimate/tools/dbt-cte-profile.ts b/packages/opencode/src/altimate/tools/dbt-cte-profile.ts index 586ce014f7..b5dc19be23 100644 --- a/packages/opencode/src/altimate/tools/dbt-cte-profile.ts +++ b/packages/opencode/src/altimate/tools/dbt-cte-profile.ts @@ -112,10 +112,38 @@ async function projectDir(): Promise { return getDbtProjectRoot() } -// True when a Jinja tag (comments and quoted strings inside it left out) calls is_incremental() or names this. +// The Jinja tags of a query, found by a scan that steps over quoted strings so a `}}` inside one does not end the tag early. +function jinjaTags(sql: string): string[] { + const text = sql.replace(/\{#[\s\S]*?#\}/g, "").replace(/\{%-?\s*raw\s*-?%\}[\s\S]*?\{%-?\s*endraw\s*-?%\}/g, "") + const tags: string[] = [] + for (let i = 0; i < text.length - 1; i++) { + const open = text.substr(i, 2) + if (open !== "{{" && open !== "{%") continue + const close = open === "{{" ? "}}" : "%}" + let j = i + 2 + while (j < text.length && !text.startsWith(close, j)) { + const ch = text[j] + if (ch === "'" || ch === '"') { + j++ + while (j < text.length && text[j] !== ch) j += text[j] === "\\" ? 2 : 1 + } + j++ + } + tags.push(text.substring(i, j)) + i = j + 1 + } + return tags +} + +// True when a Jinja tag (quoted strings inside it left out) calls is_incremental() or names this. function readsModelContext(sql: string): boolean { - const tags = sql.replace(/\{#[\s\S]*?#\}/g, "").match(/\{\{[\s\S]*?\}\}|\{%[\s\S]*?%\}/g) ?? [] - return tags.some((tag) => /\bis_incremental\s*\(|\bthis\b/.test(tag.replace(/'(?:[^'\\]|\\.)*'|"(?:[^"\\]|\\.)*"/g, "''"))) + return jinjaTags(sql).some((tag) => /\bis_incremental\s*\(|\bthis\b/.test(tag.replace(/'(?:[^'\\]|\\.)*'|"(?:[^"\\]|\\.)*"/g, "''"))) +} + +// The CTEs a typed name picks: those spelled exactly so, else those that differ only in letter case. +function nameMatches(list: CteInfo[], typed: string): CteInfo[] { + const exact = list.filter((c) => bareName(c.name) === typed) + return exact.length > 0 ? exact : list.filter((c) => bareName(c.name).toLowerCase() === typed.toLowerCase()) } function failure(title: string, output: string, extra: Record = {}) { @@ -124,7 +152,7 @@ function failure(title: string, output: string, extra: Record = export const DbtCteProfileTool = Tool.define("dbt_cte_profile", { description: - "List a dbt model's CTEs, or time one CTE with a row-count query. Without `cte` or `position` it returns the CTE names in source order (name, line, position) and runs nothing, so a read-only call works. With one of them it runs `SELECT COUNT(*)` over that CTE and the CTEs before it in the same WITH clause on the warehouse, and returns the elapsed milliseconds and the row count; that run needs write permission (`--allow-write`) because it starts the dbt adapter and compiles the model. Each run recomputes the earlier CTEs, so a model of N CTEs costs N queries. Only the model's own top-level WITH clause is read: a WITH inside a CTE body is part of that CTE.", + "List a dbt model's CTEs, or time one CTE with a row-count query. Without `cte` or `position` it returns the CTE names in source order (name, line, position) and runs nothing, so a read-only call works. With one of them it runs `SELECT COUNT(*)` over that CTE and the CTEs before it in the same WITH clause on the warehouse, and returns the elapsed milliseconds and the row count; that run needs write permission (`--allow-write`) because it starts the dbt adapter and compiles the model. Each run recomputes the earlier CTEs, so a model of N CTEs costs N queries. Only the model's own top-level WITH clause is read: a WITH inside a CTE body is part of that CTE. The list stops at a CTE whose name Jinja builds, and `incomplete` is set in the metadata then; a CTE that reads is_incremental() or this is not run, because the count runs outside the model.", parameters: z.object({ model: z .string() @@ -185,14 +213,12 @@ export const DbtCteProfileTool = Tool.define("dbt_cte_profile", { if (!target) return failure("CTE not found", `${args.model} has ${ctes.length} CTEs: there is none at position ${args.position}.${incompleteNote}`, { model: args.model }) // The name as typed wins here too: another CTE spelled exactly so is the one meant, whatever the case of this one. const typed = args.cte === undefined ? undefined : bareName(args.cte) - const spelledSo = typed === undefined ? false : ctes.some((c) => bareName(c.name) === typed) - if (typed !== undefined && (spelledSo ? bareName(target.name) !== typed : bareName(target.name).toLowerCase() !== typed.toLowerCase())) + if (typed !== undefined && !nameMatches(ctes, typed).includes(target)) return failure("CTE not found", `The CTE at position ${args.position} of ${args.model} is ${stripColumnList(target.name)}, not ${args.cte}.`, { model: args.model }) } else { // The name as typed wins; letter case only decides when nothing matches it exactly. const wanted = bareName(args.cte ?? "") - const exact = ctes.filter((c) => bareName(c.name) === wanted) - const matches = exact.length > 0 ? exact : ctes.filter((c) => bareName(c.name).toLowerCase() === wanted.toLowerCase()) + const matches = nameMatches(ctes, wanted) if (matches.length === 0) return failure("CTE not found", `${args.model} has no CTE named ${args.cte}${names ? `; it has: ${names}` : ""}.${incompleteNote}`, { model: args.model }) if (matches.length > 1) { @@ -217,7 +243,7 @@ export const DbtCteProfileTool = Tool.define("dbt_cte_profile", { if (block) return failure("no query", `${name} is inside a Jinja {% ${block} %} block of ${args.model}, which cannot be cut off at that CTE and stay valid: compile the model and profile the compiled SQL, or move the WITH out of the block.`, { model: args.model, cte: name }) // The adapter runs the query as an inline node, not as the model, so is_incremental() and this would resolve for that node instead. - if (readsModelContext(query)) return failure("not run", `${name} depends on is_incremental() or this in ${args.model}. A count query runs outside the model, so those would resolve differently and the count would not match what the model reads. Compile the model and profile the compiled SQL instead.`, { model: args.model, cte: name }) + if (readsModelContext(query)) return failure("not run", `${name} depends on is_incremental() or this in ${args.model}. Where the count runs as an inline node (the CLI and Fusion modes), those resolve for that node, not the model, so the count would not match what the model reads. Compile the model and profile the compiled SQL instead.`, { model: args.model, cte: name }) const access = await dbtAdapterFor(ctx) if (!access.adapter) return failure("not run", access.reason ?? "the dbt adapter is not available", { model: args.model, cte: name, refusal: access.refusal }) diff --git a/packages/opencode/test/altimate/dbt-cte-profile.test.ts b/packages/opencode/test/altimate/dbt-cte-profile.test.ts index b2f6d491c0..888bda7772 100644 --- a/packages/opencode/test/altimate/dbt-cte-profile.test.ts +++ b/packages/opencode/test/altimate/dbt-cte-profile.test.ts @@ -207,6 +207,20 @@ describe("dbt_cte_profile list", () => { expect(result.metadata.success).toBe(true) }) + test("a {{ this }} inside a quoted config() hook does not stop the run, a {% raw %} block is skipped, and a real this after them still does", async () => { + const tool = await initTool(DbtCteProfileTool) + const body = "with a as (select 1 as x), b as (select * from a) select * from b" + for (const [model, sql, ok] of [ + ["hook_this", `{{ config(post_hook="grant select on {{ this }} to role reporter") }}\n${body}`, true], + ["raw_this", `{% raw %}{{ this }}{% endraw %}\n${body}`, true], + ["hook_then_this", `{{ config(post_hook="grant on {{ this }}") }}\nwith a as (select * from {{ this }}), b as (select * from a) select * from b`, false], + ] as const) { + await writeFile(path.join(project, "models", "marts", `${model}.sql`), sql) + const result = await tool.execute({ model, cte: "b" }, ctx) + expect(result.metadata.success).toBe(ok) + } + }) + test("a model without CTEs lists none", async () => { const tool = await initTool(DbtCteProfileTool) const result = await tool.execute({ model: "plain" }, ctx) From 8676ac5c9f40e2ab293d2418f63797c2f4dea1e5 Mon Sep 17 00:00:00 2001 From: Harish Kumar Date: Tue, 17 Nov 2026 13:59:37 +0530 Subject: [PATCH 24/26] test: the this-in-a-hook refusal case asserts the refusal message Co-Authored-By: Claude Sonnet 5.5 --- packages/opencode/test/altimate/dbt-cte-profile.test.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/opencode/test/altimate/dbt-cte-profile.test.ts b/packages/opencode/test/altimate/dbt-cte-profile.test.ts index 888bda7772..002b1193fc 100644 --- a/packages/opencode/test/altimate/dbt-cte-profile.test.ts +++ b/packages/opencode/test/altimate/dbt-cte-profile.test.ts @@ -218,6 +218,7 @@ describe("dbt_cte_profile list", () => { await writeFile(path.join(project, "models", "marts", `${model}.sql`), sql) const result = await tool.execute({ model, cte: "b" }, ctx) expect(result.metadata.success).toBe(ok) + if (!ok) expect(result.output).toContain("is_incremental() or this") } }) From 01781b1bbba4f9d669bf7f3aed448a9450c6f299 Mon Sep 17 00:00:00 2001 From: Harish Kumar Date: Tue, 17 Nov 2026 14:07:12 +0530 Subject: [PATCH 25/26] docs: dbt_cte_profile's description says macros a CTE calls are not opened for is_incremental() Co-Authored-By: Claude Sonnet 5.5 --- packages/opencode/src/altimate/tools/dbt-cte-profile.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/opencode/src/altimate/tools/dbt-cte-profile.ts b/packages/opencode/src/altimate/tools/dbt-cte-profile.ts index b5dc19be23..b2eacea6c8 100644 --- a/packages/opencode/src/altimate/tools/dbt-cte-profile.ts +++ b/packages/opencode/src/altimate/tools/dbt-cte-profile.ts @@ -152,7 +152,7 @@ function failure(title: string, output: string, extra: Record = export const DbtCteProfileTool = Tool.define("dbt_cte_profile", { description: - "List a dbt model's CTEs, or time one CTE with a row-count query. Without `cte` or `position` it returns the CTE names in source order (name, line, position) and runs nothing, so a read-only call works. With one of them it runs `SELECT COUNT(*)` over that CTE and the CTEs before it in the same WITH clause on the warehouse, and returns the elapsed milliseconds and the row count; that run needs write permission (`--allow-write`) because it starts the dbt adapter and compiles the model. Each run recomputes the earlier CTEs, so a model of N CTEs costs N queries. Only the model's own top-level WITH clause is read: a WITH inside a CTE body is part of that CTE. The list stops at a CTE whose name Jinja builds, and `incomplete` is set in the metadata then; a CTE that reads is_incremental() or this is not run, because the count runs outside the model.", + "List a dbt model's CTEs, or time one CTE with a row-count query. Without `cte` or `position` it returns the CTE names in source order (name, line, position) and runs nothing, so a read-only call works. With one of them it runs `SELECT COUNT(*)` over that CTE and the CTEs before it in the same WITH clause on the warehouse, and returns the elapsed milliseconds and the row count; that run needs write permission (`--allow-write`) because it starts the dbt adapter and compiles the model. Each run recomputes the earlier CTEs, so a model of N CTEs costs N queries. Only the model's own top-level WITH clause is read: a WITH inside a CTE body is part of that CTE. The list stops at a CTE whose name Jinja builds, and `incomplete` is set in the metadata then; a CTE that reads is_incremental() or this in its own SQL is not run, because the count runs outside the model. Macros the CTE calls are not opened: one that reads is_incremental() inside is not caught, so treat the count of such a model as a full-history count.", parameters: z.object({ model: z .string() From dab9354abafac5ed07837596c13c44887f3eba70 Mon Sep 17 00:00:00 2001 From: Harish Kumar Date: Tue, 17 Nov 2026 14:13:40 +0530 Subject: [PATCH 26/26] docs: dbt_cte_profile's description names where the is_incremental()/this check looks Co-Authored-By: Claude Sonnet 5.5 --- packages/opencode/src/altimate/tools/dbt-cte-profile.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/opencode/src/altimate/tools/dbt-cte-profile.ts b/packages/opencode/src/altimate/tools/dbt-cte-profile.ts index b2eacea6c8..512cb3851b 100644 --- a/packages/opencode/src/altimate/tools/dbt-cte-profile.ts +++ b/packages/opencode/src/altimate/tools/dbt-cte-profile.ts @@ -152,7 +152,7 @@ function failure(title: string, output: string, extra: Record = export const DbtCteProfileTool = Tool.define("dbt_cte_profile", { description: - "List a dbt model's CTEs, or time one CTE with a row-count query. Without `cte` or `position` it returns the CTE names in source order (name, line, position) and runs nothing, so a read-only call works. With one of them it runs `SELECT COUNT(*)` over that CTE and the CTEs before it in the same WITH clause on the warehouse, and returns the elapsed milliseconds and the row count; that run needs write permission (`--allow-write`) because it starts the dbt adapter and compiles the model. Each run recomputes the earlier CTEs, so a model of N CTEs costs N queries. Only the model's own top-level WITH clause is read: a WITH inside a CTE body is part of that CTE. The list stops at a CTE whose name Jinja builds, and `incomplete` is set in the metadata then; a CTE that reads is_incremental() or this in its own SQL is not run, because the count runs outside the model. Macros the CTE calls are not opened: one that reads is_incremental() inside is not caught, so treat the count of such a model as a full-history count.", + "List a dbt model's CTEs, or time one CTE with a row-count query. Without `cte` or `position` it returns the CTE names in source order (name, line, position) and runs nothing, so a read-only call works. With one of them it runs `SELECT COUNT(*)` over that CTE and the CTEs before it in the same WITH clause on the warehouse, and returns the elapsed milliseconds and the row count; that run needs write permission (`--allow-write`) because it starts the dbt adapter and compiles the model. Each run recomputes the earlier CTEs, so a model of N CTEs costs N queries. Only the model's own top-level WITH clause is read: a WITH inside a CTE body is part of that CTE. The list stops at a CTE whose name Jinja builds, and `incomplete` is set in the metadata then; a CTE whose count query reads is_incremental() or this, in its own SQL, an earlier CTE or the model header, is not run, because the count runs outside the model. Macros the query calls are not opened: one that reads is_incremental() inside is not caught, so treat the count of such a model as a full-history count.", parameters: z.object({ model: z .string()