From f84d2a7b16d6c7a1096505e5f94dbc81de39b475 Mon Sep 17 00:00:00 2001 From: Harish Kumar Date: Thu, 19 Nov 2026 23:51:10 +0530 Subject: [PATCH 1/7] fix: dbt tools keep the project's manifest, time CTEs without dbt start-up, name graph clauses, name bad arguments, and key the daemon on the Python env - The tools' dbt adapter has dbt write its artifacts to /.altimate-code (DBT_TARGET_PATH in the adapter's subprocess environment only), so a tool's compile or show no longer rewrites the project's target/manifest.json with compiled SQL for only the nodes it compiled. dbt_health still finds the project's catalog.json. - dbt_cte_profile reports queryMs as the warehouse time of the count query: dbt's own timing from the run_results.json the command wrote, else the call's time less a trivial query's time through dbt (measured once per adapter). totalMs keeps the whole call; timing says which measure was used. - dbt_sql_graph names join, filter, group and sort nodes by their clause (LEFT JOIN payments ON ..., WHERE status <> ...), cut at 80 characters; the backend's key stays the id. - A legacy zod tool without its own formatter now reports each invalid or missing field (change_type is required (expected one of ...)) instead of "SchemaError: Expected ". - The tool daemon's socket key includes VIRTUAL_ENV and CONDA_PREFIX, so activating the project's virtualenv reaches a daemon that runs dbt with it; an adapter that didn't start or load names the Python it used and how to fix it. Co-Authored-By: Claude Opus 5.5 --- docs/docs/usage/tool.md | 4 +- packages/dbt-tools/src/adapter.ts | 20 ++- .../test/adapter-target-path.test.ts | 66 ++++++++++ .../altimate/native/connections/register.ts | 29 ++++- .../opencode/src/altimate/tool-cli/daemon.ts | 6 +- .../opencode/src/altimate/tool-zod-compat.ts | 65 +++++++--- .../src/altimate/tools/dbt-adapter-access.ts | 15 ++- .../src/altimate/tools/dbt-cte-profile.ts | 109 +++++++++++++++- .../opencode/src/altimate/tools/dbt-health.ts | 11 +- .../src/altimate/tools/dbt-sql-graph.ts | 40 +++++- .../test/altimate/dbt-adapter-access.test.ts | 44 ++++++- .../test/altimate/dbt-cte-profile.test.ts | 118 +++++++++++++++++- .../opencode/test/altimate/dbt-health.test.ts | 24 +++- .../test/altimate/dbt-sql-graph.test.ts | 45 ++++++- .../test/altimate/impact-analysis.test.ts | 26 ++++ .../test/altimate/tool-cli/daemon.test.ts | 11 ++ 16 files changed, 584 insertions(+), 49 deletions(-) create mode 100644 packages/dbt-tools/test/adapter-target-path.test.ts diff --git a/docs/docs/usage/tool.md b/docs/docs/usage/tool.md index d630c6ebd0..59a4526e97 100644 --- a/docs/docs/usage/tool.md +++ b/docs/docs/usage/tool.md @@ -68,13 +68,15 @@ altimate-code tool daemon status altimate-code tool daemon stop ``` -`status` and `stop` locate the running daemon the same way `run` does: by project directory, CLI version, the `--connections` file (if any), whether workspace tools are on, and the caller's `ALTIMATE_CODE_CONN_*`, `ALTIMATE_DISABLE_SCHEMA_VALUES`, `ALTIMATE_INTEGRATIONS` and `ALTIMATE_DISABLE_WORKSPACE` environment variables. Run them with the same flags and variables as the call that started the daemon, or they will report no daemon running for this project even though one is. +`status` and `stop` locate the running daemon the same way `run` does: by project directory, CLI version, the `--connections` file (if any), whether workspace tools are on, and the caller's `ALTIMATE_CODE_CONN_*`, `ALTIMATE_DISABLE_SCHEMA_VALUES`, `ALTIMATE_INTEGRATIONS`, `ALTIMATE_DISABLE_WORKSPACE`, `VIRTUAL_ENV` and `CONDA_PREFIX` environment variables. The last two pick the Python the daemon's dbt adapter runs dbt with, so activating the project's virtualenv gets a daemon that uses it rather than one started without it. Run them with the same flags and variables as the call that started the daemon, or they will report no daemon running for this project even though one is. | Setting | Description | |---|---| | `ALTIMATE_TOOL_DAEMON=0` | Always run in-process. | | `ALTIMATE_TOOL_DAEMON_IDLE_MS` | Idle time before the daemon exits (default `600000`). | +The tools that start the dbt adapter (`dbt_lineage`, `dbt_cte_profile`, `dbt_sql_graph`, `dbt_health`, `dbt_cost_estimate`, and `sql_execute` routed through dbt) have dbt write its artifacts to `/.altimate-code/` rather than the project's own `target/`: a dbt command rewrites `manifest.json` with compiled SQL for only what it compiled, so the project's `target/manifest.json` is left as your own `dbt compile` wrote it. + The daemon listens on a local socket (a named pipe on Windows) under the Altimate Code state directory and removes it on exit. ## Claude Code routing diff --git a/packages/dbt-tools/src/adapter.ts b/packages/dbt-tools/src/adapter.ts index 706d43ad6c..4786164c1a 100644 --- a/packages/dbt-tools/src/adapter.ts +++ b/packages/dbt-tools/src/adapter.ts @@ -81,8 +81,22 @@ function terminal(): DBTTerminal { } } -function env(cfg: Config): RuntimePythonEnvironment { +/** What `create` needs besides the saved config. */ +export type CreateOptions = { + /** + * The folder dbt writes its artifacts to (manifest.json, partial_parse.msgpack, run_results.json) for + * every command this adapter runs, given to dbt as DBT_TARGET_PATH. Unset, dbt writes to the project's + * own target-path, which is what a command run for the user should do. A caller that parses and + * compiles for its own reading passes a folder of its own: dbt rewrites manifest.json on every + * command, with compiled SQL only for what that command compiled, so writing to the project's target + * would take the compiled SQL of every other model away from whoever reads that manifest. + */ + targetPath?: string +} + +function env(cfg: Config, options: CreateOptions): RuntimePythonEnvironment { const vars = { ...process.env } + if (options.targetPath) vars.DBT_TARGET_PATH = options.targetPath return { pythonPath: cfg.pythonPath, getEnvironmentVariables: () => vars, @@ -98,7 +112,7 @@ export function hasModelLineage(adapter?: DBTProjectIntegrationAdapter): boolean return typeof target.getModelLineage === "function" } -export async function create(cfg: Config): Promise { +export async function create(cfg: Config, options: CreateOptions = {}): Promise { const config = configuration(cfg) const term = terminal() const factory = new DBTCommandFactory(config) @@ -106,7 +120,7 @@ export async function create(cfg: Config): Promise const http = new AltimateHttpClient(term, config) const client = new DbtIntegrationClient(http, term) - const runtime = env(cfg) + const runtime = env(cfg, options) const provider = { getCurrentEnvironment: () => runtime, onEnvironmentChanged: () => () => {}, diff --git a/packages/dbt-tools/test/adapter-target-path.test.ts b/packages/dbt-tools/test/adapter-target-path.test.ts new file mode 100644 index 0000000000..86957353a2 --- /dev/null +++ b/packages/dbt-tools/test/adapter-target-path.test.ts @@ -0,0 +1,66 @@ +import { describe, test, expect } from "bun:test" +import { chmodSync, existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "fs" +import { tmpdir } from "os" +import { join } from "path" +import { create } from "../src/adapter" + +/** A stand-in `python` that records the DBT_TARGET_PATH each run sees and fails the bridge run. */ +function recordingPython(root: string) { + const log = join(root, "target-path.log") + const python = join(root, "python") + writeFileSync( + python, + [ + "#!/bin/sh", + `printf '%s\\n' "\${DBT_TARGET_PATH-unset}" >> '${log}'`, + 'case "$1" in', + ' --version) echo "Python 3.11.9"; exit 0 ;;', + ' -c) echo "1.10.0"; exit 0 ;;', + "esac", + "exit 3", + ].join("\n") + "\n", + ) + chmodSync(python, 0o755) + return { python, seen: () => (existsSync(log) ? readFileSync(log, "utf8").trim().split("\n") : []) } +} + +// The adapter's dbt commands rewrite manifest.json in the target folder they are given; a caller that +// compiles for its own reading gives one of its own, so the project's target/manifest.json is left alone. +describe.skipIf(process.platform === "win32")("create's target folder", () => { + const start = async (targetPath?: string) => { + const root = mkdtempSync(join(tmpdir(), "dbt-tools-target-")) + const project = join(root, "project") + mkdirSync(project) + writeFileSync(join(project, "dbt_project.yml"), "name: p\nversion: '1.0'\nprofile: p\n") + const { python, seen } = recordingPython(root) + const saved = process.env.DBT_TARGET_PATH + delete process.env.DBT_TARGET_PATH + try { + await create( + { projectRoot: project, pythonPath: python, dbtIntegration: "corecommand", queryLimit: 500 }, + targetPath ? { targetPath } : undefined, + ).then( + (adapter) => adapter.dispose(), + () => {}, + ) + // The host's own environment is not changed: other readers of DBT_TARGET_PATH still see the project's. + expect(process.env.DBT_TARGET_PATH).toBeUndefined() + return seen() + } finally { + if (saved !== undefined) process.env.DBT_TARGET_PATH = saved + rmSync(root, { recursive: true, force: true }) + } + } + + test("every dbt process the adapter starts writes to the folder it is given", async () => { + const runs = await start("/tmp/project/target/.altimate-code") + expect(runs.length).toBeGreaterThan(0) + expect(new Set(runs)).toEqual(new Set(["/tmp/project/target/.altimate-code"])) + }) + + test("without one, dbt writes where the project says", async () => { + const runs = await start() + expect(runs.length).toBeGreaterThan(0) + expect(new Set(runs)).toEqual(new Set(["unset"])) + }) +}) diff --git a/packages/opencode/src/altimate/native/connections/register.ts b/packages/opencode/src/altimate/native/connections/register.ts index b404bd0444..2f85c1cfb7 100644 --- a/packages/opencode/src/altimate/native/connections/register.ts +++ b/packages/opencode/src/altimate/native/connections/register.ts @@ -6,6 +6,7 @@ * schema.inspect */ +import path from "path" import { register } from "../dispatcher" import { listSchemaTables } from "./schema-list" import * as Registry from "./registry" @@ -57,13 +58,15 @@ let dbtAdapter: any | null | undefined = undefined let dbtAdapterInflight: Promise | undefined /** The project root the cached attempt read from the dbt config; null when it found none. */ let dbtProjectRoot: string | null = null +/** The Python the cached attempt runs dbt with; null when it found no config. */ +let dbtPythonPath: string | null = null /** Test seams for the single-flight. Production leaves `readConfig` unset. */ export const dbtAdapterInternals: { /** Replaces the dbt config read, so a test can hold an attempt open. */ readConfig?: () => Promise /** Replaces dbt-tools' adapter creation, so a test can see which integration modes were tried. */ - create?: (config: any) => Promise + create?: (config: any, options?: { targetPath?: string }) => Promise /** Replaces the check for dbt-integration's lineage, which a test's fake adapter provides itself. */ hasModelLineage?: (adapter?: unknown) => boolean /** Adapter creation attempts since the last reset — what single-flight bounds. */ @@ -109,21 +112,24 @@ async function ensureDbtAdapter(): Promise { : await readDbtConfig() if (dbtAdapterInflight === mine) { dbtProjectRoot = typeof dbtConfig?.projectRoot === "string" ? dbtConfig.projectRoot : null + dbtPythonPath = typeof dbtConfig?.pythonPath === "string" ? dbtConfig.pythonPath : null } if (!dbtConfig) return publish(null) // Check if dbt_project.yml exists const fs = await import("fs") - const path = await import("path") if ( !fs.existsSync(path.join(dbtConfig.projectRoot, "dbt_project.yml")) ) { return publish(null) } - // Create the adapter + // Create the adapter. Its dbt commands write their artifacts to a folder of their own: the project's + // target/manifest.json is the user's (and other tools'), and a tool's compile or show would rewrite + // it with compiled SQL for only the nodes that command compiled. const create = dbtAdapterInternals.create ?? (await import("../../../../../dbt-tools/src/adapter")).create - return publish(await create(dbtConfig), dbtConfig.projectRoot) + const targetPath = await dbtToolTargetPath(dbtConfig.projectRoot) + return publish(await create(dbtConfig, { targetPath }), dbtConfig.projectRoot) } catch { // dbt-tools not available or config invalid — fall back to native return publish(null) @@ -135,6 +141,15 @@ async function ensureDbtAdapter(): Promise { return mine } +/** The folder, inside the project's target folder, where the tools' dbt adapter writes its dbt artifacts. */ +export const DBT_TOOL_TARGET_DIR = ".altimate-code" + +/** Where the tools' dbt adapter has dbt write manifest.json and the rest: never the project's own target folder. */ +export async function dbtToolTargetPath(projectRoot: string): Promise { + const { resolveDbtTargetPath } = await import("../../validators/validator-utils") + return path.join(await resolveDbtTargetPath(projectRoot), DBT_TOOL_TARGET_DIR) +} + /** Where a `warehouse`-less call would actually go. */ export type DefaultTarget = | { @@ -316,6 +331,11 @@ export async function getDbtProjectRoot(): Promise { } } +/** The Python the dbt adapter runs dbt with, once an attempt has read the dbt config. */ +export function getDbtPythonPath(): string | undefined { + return dbtPythonPath ?? undefined +} + /** Reset dbt adapter (for testing). */ export function resetDbtAdapter(): void { dbtAdapter = undefined @@ -326,6 +346,7 @@ export function resetDbtAdapter(): void { dbtAdapterInternals.attempts = 0 dbtAdapterInternals.writes = 0 dbtProjectRoot = null + dbtPythonPath = null } // --------------------------------------------------------------------------- diff --git a/packages/opencode/src/altimate/tool-cli/daemon.ts b/packages/opencode/src/altimate/tool-cli/daemon.ts index e28a10f510..86deb24422 100644 --- a/packages/opencode/src/altimate/tool-cli/daemon.ts +++ b/packages/opencode/src/altimate/tool-cli/daemon.ts @@ -44,8 +44,10 @@ export function resolveDirectory(directory: string): string { const CONN_ENV_PREFIX = "ALTIMATE_CODE_CONN_" // Variables a daemon reads once at spawn that change what it returns or exposes (value sampling, integration -// routing, workspace tools), so a caller with a different value must not be served by a daemon started without it. -const BEHAVIOUR_ENV = ["ALTIMATE_DISABLE_SCHEMA_VALUES", "ALTIMATE_INTEGRATIONS", "ALTIMATE_DISABLE_WORKSPACE"] +// routing, workspace tools, and the Python environment its dbt adapter runs dbt with), so a caller with a +// different value must not be served by a daemon started without it: a daemon started outside the project's +// virtualenv would otherwise go on answering that dbt didn't load after the caller activated it. +const BEHAVIOUR_ENV = ["ALTIMATE_DISABLE_SCHEMA_VALUES", "ALTIMATE_INTEGRATIONS", "ALTIMATE_DISABLE_WORKSPACE", "VIRTUAL_ENV", "CONDA_PREFIX"] /** A short digest of the caller's `ALTIMATE_CODE_CONN_*` variables and the behaviour variables above, empty when * none are set. A daemon reads them once at spawn, so callers with different values must not share one. */ diff --git a/packages/opencode/src/altimate/tool-zod-compat.ts b/packages/opencode/src/altimate/tool-zod-compat.ts index ccab23a3cd..15855e8380 100644 --- a/packages/opencode/src/altimate/tool-zod-compat.ts +++ b/packages/opencode/src/altimate/tool-zod-compat.ts @@ -176,6 +176,35 @@ export function zodToJsonSchema(schema: z.ZodType): JSONSchema7 { : rest) as JSONSchema7 } +// altimate_change start — argument errors that name the field +/** The value at `path` in `input`, undefined when any step is missing. */ +function valueAt(input: unknown, path: readonly PropertyKey[]): unknown { + let value = input + for (const key of path) { + if (typeof value !== "object" || value === null || !Object.hasOwn(value, key)) return undefined + value = (value as Record)[key] + } + return value +} + +/** + * Each issue of a failed zod parse as one clause that names its field: `change_type is required (expected + * one of "remove"|"rename")` for a missing one, `limit: Too small: expected number to be >=1` for a wrong one. + */ +export function describeZodIssues(error: z.ZodError, input: unknown): string { + const clauses = error.issues.map((issue) => { + if (issue.path.length === 0) return issue.message + const field = issue.path.map((p, i) => (typeof p === "number" ? `[${p}]` : `${i === 0 ? "" : "."}${String(p)}`)).join("") + if (valueAt(input, issue.path) === undefined) { + const expected = issue.message.replace(/^Invalid (?:input|option):\s*/, "").replace(/,\s*received undefined$/, "") + return `${field} is required (${expected})` + } + return `${field}: ${issue.message}` + }) + return clauses.length > 0 ? clauses.join("; ") : error.message +} +// altimate_change end + /** Build a `DefWithoutID` from a resolved legacy tool object (no Effect wrapper). */ function legacyDefToDef(legacy: LegacyToolDef): DefWithoutID { const validationSchema = zodToJsonSchema(legacy.parameters) @@ -200,24 +229,24 @@ function legacyDefToDef(legacy: LegacyToolDef): DefWithoutID { // tool needs hidden compatibility arguments that should not be advertised. jsonSchema: legacy.jsonSchema ?? validationSchema, // altimate_change end - ...(legacy.formatValidationError - ? { - // altimate_change start — the new Tool API decodes via Schema.declare, so the - // error reaching us is an Effect SchemaIssue (no zod `.issues`/`.path`), not a - // z.ZodError. Recover the original input from Effect SchemaError.issue.actual and - // re-run the zod schema to produce the real z.ZodError the legacy formatter - // expects; fall back to the original error if recovery is impossible. - formatValidationError: (error: unknown) => { - const actual = schemaErrorActual(error) - if (actual.found) { - const parsed = legacy.parameters.safeParse(actual.value) - if (!parsed.success) return legacy.formatValidationError!(parsed.error) - } - return legacy.formatValidationError!(error as z.ZodError) - }, - // altimate_change end - } - : {}), + // altimate_change start — the new Tool API decodes via Schema.declare, so the error reaching + // us is an Effect SchemaIssue (no zod `.issues`/`.path`), not a z.ZodError: on its own it reads + // "Expected " and names no field. Recover the original input from Effect + // SchemaError.issue.actual and re-run the zod schema to produce the real z.ZodError, for the + // legacy tool's own formatter or, when it has none, one that names each field and what it + // expects; fall back to the original error if recovery is impossible. + formatValidationError: (error: unknown) => { + const actual = schemaErrorActual(error) + if (actual.found) { + const parsed = legacy.parameters.safeParse(actual.value) + if (!parsed.success) + return legacy.formatValidationError + ? legacy.formatValidationError(parsed.error) + : describeZodIssues(parsed.error, actual.value) + } + return legacy.formatValidationError ? legacy.formatValidationError(error as z.ZodError) : String(error) + }, + // altimate_change end execute: (args: unknown, ctx: NewContext) => // altimate_change start — bridge the ambient Instance ALS into the Effect InstanceRef so // legacy tools whose execute calls async facades (Skill.get, Agent.list, …) — which resolve diff --git a/packages/opencode/src/altimate/tools/dbt-adapter-access.ts b/packages/opencode/src/altimate/tools/dbt-adapter-access.ts index 8c81cb0980..a808ae8845 100644 --- a/packages/opencode/src/altimate/tools/dbt-adapter-access.ts +++ b/packages/opencode/src/altimate/tools/dbt-adapter-access.ts @@ -55,7 +55,7 @@ export async function dbtAdapterFor(ctx: Pick 0 ? exact : list.filter((c) => bareName(c.name).toLowerCase() === typed.toLowerCase()) } +/** Test seams. */ +export const dbtCteProfileInternals = { + /** The trivial query whose time through dbt stands for dbt's own start-up, when dbt reports no timing. */ + baselineQuery: "select 1 as _profile_baseline", +} + +/** Per adapter: how long a trivial query takes through it, dbt's start-up with next to no warehouse work. */ +const baselines = new WeakMap>() + +/** A dbt timestamp in ms; dbt writes UTC, with or without the zone. */ +function utc(value: unknown): number { + if (typeof value !== "string" || !value) return NaN + return Date.parse(/(Z|[+-]\d\d:?\d\d)$/i.test(value) ? value : `${value}Z`) +} + +/** The run_results.json dbt wrote for the count query, as the warehouse time of its execute step, in ms. */ +export function dbtQueryMs(targetDir: string, startedAt: number, compiledSql?: unknown): number | undefined { + let raw: any + try { + raw = JSON.parse(fs.readFileSync(path.join(targetDir, "run_results.json"), "utf-8")) + } catch { + return undefined + } + // Only the file this call's command wrote: a later start than the call, a `show`, one inline node, + // and its compiled SQL inside what the adapter ran, when it says (another command may have run meanwhile). + const invoked = utc(raw?.metadata?.invocation_started_at ?? raw?.metadata?.generated_at) + if (!Number.isFinite(invoked) || invoked < startedAt) return undefined + if (raw?.args?.which !== undefined && raw.args.which !== "show") return undefined + const results = Array.isArray(raw?.results) ? raw.results : [] + if (results.length !== 1) return undefined + const node = results[0] + // The adapter's SQL is what ran: the node's compiled code inside dbt's query comment and limit. + if (typeof compiledSql === "string" && typeof node?.compiled_code === "string" && !compiledSql.includes(node.compiled_code.trim())) return undefined + const execute = Array.isArray(node?.timing) ? node.timing.find((t: any) => t?.name === "execute") : undefined + const span = utc(execute?.completed_at) - utc(execute?.started_at) + if (Number.isFinite(span) && span >= 0) return Math.round(span) + return typeof node?.execution_time === "number" && node.execution_time >= 0 ? Math.round(node.execution_time * 1000) : undefined +} + +type QueryTime = { source: "dbt" | "baseline" | "wall"; queryMs: number; startupMs?: number } + +/** + * The count query's time on the warehouse, without dbt's start-up, which a command-line dbt pays on every + * query (seconds, against milliseconds for a small CTE). First dbt's own timing of the query, from the + * run_results.json it wrote; else the call's time less a trivial query's time through the same adapter, + * measured once per adapter; else the call's time as is. + */ +async function queryTime(adapter: any, call: { startedAt: number; totalMs: number; compiledSql?: unknown }, ctx: Pick): Promise { + const dir = typeof adapter.getTargetPath === "function" ? adapter.getTargetPath() : undefined + const fromDbt = typeof dir === "string" && dir ? dbtQueryMs(dir, call.startedAt, call.compiledSql) : undefined + if (fromDbt !== undefined) return { source: "dbt", queryMs: Math.min(fromDbt, call.totalMs) } + let baseline = baselines.get(adapter) + if (!baseline) { + baseline = (async () => { + const started = performance.now() + try { + await executeQuery({ type: "dbt" }, {}, { signal: ctx.abort, sessionID: ctx.sessionID }, () => + adapter.immediatelyExecuteSQLWithLimit(dbtCteProfileInternals.baselineQuery, "", 1), + ) + return Math.round(performance.now() - started) + } catch { + return undefined + } + })() + baselines.set(adapter, baseline) + } + const startupMs = await baseline + // A failed or cancelled measurement is not kept: the next call measures again. + if (startupMs === undefined) { + baselines.delete(adapter) + ctx.abort.throwIfAborted() + return { source: "wall", queryMs: call.totalMs } + } + const startup = Math.min(startupMs, call.totalMs) + return { source: "baseline", queryMs: call.totalMs - startup, startupMs: startup } +} + function failure(title: string, output: string, extra: Record = {}) { return { title: `dbt_cte_profile: ${title}`, metadata: { success: false, error: output, ...extra }, output } } export const DbtCteProfileTool = Tool.define("dbt_cte_profile", { description: - "List a dbt model's CTEs, or time one CTE with a row-count query. Without `cte` or `position` it returns the CTE names in source order (name, line, position) and runs nothing, so a read-only call works. With one of them it runs `SELECT COUNT(*)` over that CTE and the CTEs before it in the same WITH clause on the warehouse, and returns the elapsed milliseconds and the row count; that run needs write permission (`--allow-write`) because it starts the dbt adapter and compiles the model. Each run recomputes the earlier CTEs, so a model of N CTEs costs N queries. Only the model's own top-level WITH clause is read: a WITH inside a CTE body is part of that CTE. The list stops at a CTE whose name Jinja builds, and `incomplete` is set in the metadata then; a CTE whose count query reads is_incremental() or this, in its own SQL, an earlier CTE or the model header, is not run, because the count runs outside the model. Macros the query calls are not opened: one that reads is_incremental() inside is not caught, so treat the count of such a model as a full-history count.", + "List a dbt model's CTEs, or time one CTE with a row-count query. Without `cte` or `position` it returns the CTE names in source order (name, line, position) and runs nothing, so a read-only call works. With one of them it runs `SELECT COUNT(*)` over that CTE and the CTEs before it in the same WITH clause on the warehouse, and returns the row count and the query's milliseconds on the warehouse (`queryMs`): dbt's own timing of the query's execute step where dbt reports one (run_results.json), else the call's time less that of a trivial query through dbt, so a command-line dbt's start-up of a second or more per query is not counted; `totalMs` is the whole call, start-up included, and `timing` says which measure `queryMs` is (dbt, baseline, or wall when neither was available); that run needs write permission (`--allow-write`) because it starts the dbt adapter and compiles the model. Each run recomputes the earlier CTEs, so a model of N CTEs costs N queries. Only the model's own top-level WITH clause is read: a WITH inside a CTE body is part of that CTE. The list stops at a CTE whose name Jinja builds, and `incomplete` is set in the metadata then; a CTE whose count query reads is_incremental() or this, in its own SQL, an earlier CTE or the model header, is not run, because the count runs outside the model. Macros the query calls are not opened: one that reads is_incremental() inside is not caught, so treat the count of such a model as a full-history count.", parameters: z.object({ model: z .string() @@ -249,18 +326,38 @@ export const DbtCteProfileTool = Tool.define("dbt_cte_profile", { if (!access.adapter) return failure("not run", access.reason ?? "the dbt adapter is not available", { model: args.model, cte: name, refusal: access.refusal }) const adapter = access.adapter ctx.abort.throwIfAborted() + const startedAt = Date.now() const started = performance.now() try { // The adapter has no cancel of its own: the run's timeout, budget and abort end the wait, not the query. - const result: { data?: Record[] } = await executeQuery({ type: "dbt" }, {}, { signal: ctx.abort, sessionID: ctx.sessionID }, () => + const result: { data?: Record[]; compiledSql?: unknown } = await executeQuery({ type: "dbt" }, {}, { signal: ctx.abort, sessionID: ctx.sessionID }, () => adapter.immediatelyExecuteSQLWithLimit(query, path.basename(modelPath, ".sql"), 1), ) - const queryMs = Math.round(performance.now() - started) + const totalMs = Math.round(performance.now() - started) const rows = extractRowCount(result?.data) + const timing = await queryTime(adapter, { startedAt, totalMs, compiledSql: result?.compiledSql }, ctx) + const rowText = `${rows} row${rows === 1 ? "" : "s"}` + const output = + timing.source === "dbt" + ? `${name}: ${timing.queryMs} ms on the warehouse (dbt's own timing of the query), ${rowText}; ${totalMs} ms in all with dbt's start-up` + : timing.source === "baseline" + ? `${name}: ${timing.queryMs} ms on the warehouse (${totalMs} ms in all, less ${timing.startupMs} ms that a trivial query takes through dbt), ${rowText}` + : `${name}: ${totalMs} ms, ${rowText} (dbt's start-up included: no query timing of its own was available)` return { - title: `dbt_cte_profile: ${name} ${queryMs} ms`, - metadata: { success: true, model: args.model, cte: name, index: target.index, position: ctes.indexOf(target), queryMs, rows }, - output: `${name}: ${queryMs} ms, ${rows} row${rows === 1 ? "" : "s"}`, + title: `dbt_cte_profile: ${name} ${timing.queryMs} ms`, + metadata: { + success: true, + model: args.model, + cte: name, + index: target.index, + position: ctes.indexOf(target), + queryMs: timing.queryMs, + totalMs, + timing: timing.source, + ...(timing.startupMs !== undefined && { startupMs: timing.startupMs }), + rows, + }, + output, } } catch (e) { if (ctx.abort.aborted) throw e diff --git a/packages/opencode/src/altimate/tools/dbt-health.ts b/packages/opencode/src/altimate/tools/dbt-health.ts index b91c5d883d..6ca9f8541c 100644 --- a/packages/opencode/src/altimate/tools/dbt-health.ts +++ b/packages/opencode/src/altimate/tools/dbt-health.ts @@ -267,7 +267,7 @@ function failure(title: string, output: string, extra: Record = export const DbtHealthTool = Tool.define("dbt_health", { description: - "DataPilot project health insights for the models of the dbt project (hard-coded references, missing tests or documentation, naming and structure issues, and so on). Uses the project's dbt integration (starts it, so it needs write permission) for a fresh manifest, then runs DataPilot (the `altimate-datapilot-cli` Python package, which must be installed in the project's Python) over the manifest, and the catalog when target/catalog.json exists. Returns `insights`: uniqueId, path, name, severity, message and recommendation for each, plus the model's absolute `file` and `properties` file for the project's own models. When dbt can't parse the project the answer is a failure with the parse errors (no insights from the last manifest); a failed project check alone is answered from the last good manifest, labelled `stale_manifest`. A very large result is cut at a byte limit: `truncated` is then true, and `total` counts every insight.", + "DataPilot project health insights for the models of the dbt project (hard-coded references, missing tests or documentation, naming and structure issues, and so on). Uses the project's dbt integration (starts it, so it needs write permission) for a fresh manifest, then runs DataPilot (the `altimate-datapilot-cli` Python package, which must be installed in the project's Python) over the manifest, and the catalog when the project's target/catalog.json exists. Returns `insights`: uniqueId, path, name, severity, message and recommendation for each, plus the model's absolute `file` and `properties` file for the project's own models. When dbt can't parse the project the answer is a failure with the parse errors (no insights from the last manifest); a failed project check alone is answered from the last good manifest, labelled `stale_manifest`. A very large result is cut at a byte limit: `truncated` is then true, and `total` counts every insight.", parameters: z.object({ config_path: z .string() @@ -346,8 +346,13 @@ export const DbtHealthTool = Tool.define("dbt_health", { age_seconds: Math.max(0, Math.round((Date.now() - written) / 1000)), } } - const catalogFile = path.join(path.dirname(manifest), "catalog.json") - const catalog = fs.existsSync(catalogFile) ? catalogFile : "" + // The adapter's dbt writes to a target folder of its own, where `dbt docs generate` never puts a catalog: + // the one beside its manifest, else the project's own. + const { resolveDbtTargetPath } = await import("../validators/validator-utils") + const catalog = + [path.join(path.dirname(manifest), "catalog.json"), path.join(await resolveDbtTargetPath(root), "catalog.json")].find((file) => + fs.existsSync(file), + ) ?? "" let configPath = "" if (args.config_path) { configPath = path.resolve(root, args.config_path) diff --git a/packages/opencode/src/altimate/tools/dbt-sql-graph.ts b/packages/opencode/src/altimate/tools/dbt-sql-graph.ts index c3a950902c..7db2a6edb0 100644 --- a/packages/opencode/src/altimate/tools/dbt-sql-graph.ts +++ b/packages/opencode/src/altimate/tools/dbt-sql-graph.ts @@ -117,6 +117,41 @@ export function mergeColumns( return Object.fromEntries(merged) } +/** Longest name a join, filter, group or sort node is given; its full expression stays in `sql`. */ +export const CLAUSE_NAME_CHARS = 80 + +/** An identifier as written, without its quotes. */ +function unquote(part: string): string { + const q = part[0] + if ((q === '"' || q === "`") && part.endsWith(q) && part.length > 1) return part.slice(1, -1).replaceAll(q + q, q) + if (q === "[" && part.endsWith("]")) return part.slice(1, -1) + return part +} + +const IDENT = String.raw`(?:"(?:[^"]|"")+"|\`(?:[^\`]|\`\`)+\`|\[[^\]]+\]|[A-Za-z_][\w$]*)` +// A join's own relation, right after its JOIN keyword: `JOIN "db"."schema"."table" AS alias`. +const JOINED_RELATION = new RegExp(String.raw`^((?:[A-Za-z]+\s+)*JOIN\s+)(${IDENT}(?:\s*\.\s*${IDENT})*)(?:\s+AS)?\s+(?!ON\b|USING\b)(${IDENT})(?=\s|$)`, "i") + +/** + * A name for a node the backend leaves unnamed (a join, filter, group or sort): its clause on one line, a relation + * written as its own name (`LEFT JOIN payments ON …`, `WHERE status <> 'returned'`), cut at CLAUSE_NAME_CHARS. + * Undefined when the node has no clause to show. + */ +export function clauseName(details: Pick): string | undefined { + let text = (details.expression ?? details.sql ?? "").replace(/\s+/g, " ").trim() + if (!text) return undefined + // A joined relation keeps its last name part; its alias goes when it only repeats that name. Nothing else in + // the clause is rewritten: an `AS` in a condition (a CAST) is not a relation. + text = text.replace(JOINED_RELATION, (_all, join: string, relation: string, alias: string) => { + const last = unquote(relation.match(new RegExp(IDENT, "g"))!.at(-1)!) + return `${join}${last.toLowerCase() === unquote(alias).toLowerCase() ? last : `${last} AS ${unquote(alias)}`}` + }) + // The backend writes an inner join as a bare JOIN; its type says which join it is. + const kind = /^([A-Z]+)_JOIN$/i.exec(details.type ?? "")?.[1] + if (kind && /^JOIN\b/i.test(text)) text = `${kind.toUpperCase()} ${text}` + return text.length > CLAUSE_NAME_CHARS ? `${text.slice(0, CLAUSE_NAME_CHARS - 1).trimEnd()}…` : text +} + /** * The backend names the model's own select `__final_select__`: name it after the model (or ` (final select)` when the SQL has a node of that name), tie each * relation node to the dbt node it stands for, and drop the layout positions the list doesn't use. @@ -177,9 +212,10 @@ export function remapSqlLineage( const nodes: SqlGraphNode[] = Object.entries(details).map(([key, d]) => { const merged = { ...d, ...mapping[key] } return { - // The key is the node's identity (edges use it); the name is only what is shown. + // The key is the node's identity (edges use it); the name is only what is shown. A join, filter, group + // or sort has no name from the backend, only a generated key (`Where_1405…`): it is named by its clause. id: key, - name: key === finalKey ? finalKey : (merged.name ?? key), + name: key === finalKey ? finalKey : (merged.name ?? clauseName(merged) ?? key), type: merged.type, ...(merged.nodeId ? { nodeId: merged.nodeId } : {}), sql: merged.sql ?? merged.expression ?? "", diff --git a/packages/opencode/test/altimate/dbt-adapter-access.test.ts b/packages/opencode/test/altimate/dbt-adapter-access.test.ts index 3f47caa7f6..7611e873cf 100644 --- a/packages/opencode/test/altimate/dbt-adapter-access.test.ts +++ b/packages/opencode/test/altimate/dbt-adapter-access.test.ts @@ -9,7 +9,7 @@ import path from "node:path" import { Effect } from "effect" import type { Tool } from "@/tool/tool" import { SessionID, MessageID } from "@/session/schema" -import { dbtAdapterInternals, resetDbtAdapter } from "@/altimate/native/connections/register" +import { DBT_TOOL_TARGET_DIR, dbtAdapterInternals, dbtToolTargetPath, resetDbtAdapter } from "@/altimate/native/connections/register" import { dbtAdapterAccessInternals, dbtAdapterFor } from "@/altimate/tools/dbt-adapter-access" import { resultFailed } from "@/altimate/mcp-server/adapter" import { EXPOSED, MCP_OPEN_WRITE_TOOLS, WRITE_TOOLS } from "@/altimate/tool-cli/allow-list" @@ -180,7 +180,43 @@ describe("dbtAdapterFor", () => { test("says when the adapter didn't load the project", async () => { dbtAdapterInternals.create = async () => ({ isInitialized: () => false, dispose: async () => {} }) const access = await dbtAdapterFor(ctx) - expect(access).toEqual({ refusal: "not_loaded", reason: "the dbt adapter didn't load the project" }) + expect(access.refusal).toBe("not_loaded") + expect(access.reason).toStartWith("the dbt adapter didn't load the project: ") + }) + + test("a refusal for an adapter that didn't load names the Python dbt ran with and how to fix it", async () => { + dbtAdapterInternals.create = async () => ({ isInitialized: () => false, dispose: async () => {} }) + const access = await dbtAdapterFor(ctx) + // The Python the config chose: dbt missing from it is the usual cause. + expect(access.reason).toContain("(python3)") + expect(access.reason).toContain("Activate the project's virtualenv") + expect(access.reason).toContain("altimate-code tool daemon stop") + }) + + test("the tools' adapter is created with a dbt target folder of its own, inside the project's target", async () => { + let options: { targetPath?: string } | undefined + dbtAdapterInternals.create = async (_config, given) => { + options = given + return { isInitialized: () => true, dispose: async () => {} } + } + await dbtAdapterFor(ctx) + expect(options).toEqual({ targetPath: path.join(project, "target", DBT_TOOL_TARGET_DIR) }) + }) + + test("the tools' target folder follows the project's own target-path and DBT_TARGET_PATH", async () => { + const yml = path.join(project, "dbt_project.yml") + const saved = readFileSync(yml, "utf8") + const env = process.env.DBT_TARGET_PATH + try { + await writeFile(yml, `${saved}target-path: "build"\n`) + expect(await dbtToolTargetPath(project)).toBe(path.join(project, "build", DBT_TOOL_TARGET_DIR)) + process.env.DBT_TARGET_PATH = path.join(project, "elsewhere") + expect(await dbtToolTargetPath(project)).toBe(path.join(project, "elsewhere", DBT_TOOL_TARGET_DIR)) + } finally { + await writeFile(yml, saved) + if (env === undefined) delete process.env.DBT_TARGET_PATH + else process.env.DBT_TARGET_PATH = env + } }) test("says there is no dbt project when the config finds none", async () => { @@ -207,7 +243,9 @@ describe("dbtAdapterFor", () => { throw new Error("dbt-integration missing") } const access = await dbtAdapterFor(ctx) - expect(access).toEqual({ refusal: "not_started", reason: "the dbt adapter didn't start" }) + expect(access.refusal).toBe("not_started") + expect(access.reason).toStartWith("the dbt adapter didn't start: ") + expect(access.reason).toContain("(python3)") expect(created).toBe(1) }) diff --git a/packages/opencode/test/altimate/dbt-cte-profile.test.ts b/packages/opencode/test/altimate/dbt-cte-profile.test.ts index 002b1193fc..b378368600 100644 --- a/packages/opencode/test/altimate/dbt-cte-profile.test.ts +++ b/packages/opencode/test/altimate/dbt-cte-profile.test.ts @@ -8,7 +8,7 @@ import { Effect } from "effect" import type { Tool } from "@/tool/tool" import { SessionID, MessageID } from "@/session/schema" import { dbtAdapterInternals, resetDbtAdapter } from "@/altimate/native/connections/register" -import { DbtCteProfileTool } from "@/altimate/tools/dbt-cte-profile" +import { DbtCteProfileTool, dbtCteProfileInternals, dbtQueryMs } from "@/altimate/tools/dbt-cte-profile" import { initTool } from "./tool-fixture" const ctx: Tool.Context = { @@ -41,7 +41,11 @@ select * from final let project: string let queries: { sql: string; name: string; limit: number }[] +/** The trivial queries the tool times dbt's start-up with, kept apart from the count queries. */ +let baselines: { sql: string; name: string; limit: number }[] let respond: (sql: string) => Promise +/** The adapter's dbt target folder, where a real dbt writes run_results.json; undefined: the adapter has none. */ +let adapterTarget: string | undefined function useAdapter() { dbtAdapterInternals.readConfig = async () => ({ @@ -53,8 +57,9 @@ function useAdapter() { dbtAdapterInternals.create = async () => ({ isInitialized: () => true, dispose: async () => {}, + getTargetPath: () => adapterTarget, immediatelyExecuteSQLWithLimit: async (sql: string, name: string, limit: number) => { - queries.push({ sql, name, limit }) + ;(sql === dbtCteProfileInternals.baselineQuery ? baselines : queries).push({ sql, name, limit }) return respond(sql) }, }) @@ -104,7 +109,9 @@ afterAll(async () => { }) beforeEach(() => { + adapterTarget = undefined queries = [] + baselines = [] respond = async () => ({ columnNames: ["_PROFILE_COUNT"], data: [{ _PROFILE_COUNT: 42 }] }) resetDbtAdapter() useAdapter() @@ -505,3 +512,110 @@ describe("dbt_cte_profile run", () => { expect(queries).toEqual([]) }) }) + +/** What dbt writes for one `dbt show --inline`: its start, and the execute step's span. */ +function runResults(opts: { started: Date; executeMs: number; compiled: string; which?: string }) { + const iso = (d: Date) => d.toISOString().replace("Z", "123Z") + const exec0 = new Date(opts.started.getTime() + 1500) + return JSON.stringify({ + metadata: { invocation_started_at: iso(opts.started), generated_at: iso(new Date(exec0.getTime() + opts.executeMs + 5)) }, + args: { which: opts.which ?? "show" }, + results: [ + { + unique_id: "sql_operation.jaffle_shop.inline_query", + execution_time: (opts.executeMs + 2) / 1000, + compiled_code: opts.compiled, + timing: [ + { name: "compile", started_at: iso(new Date(exec0.getTime() - 2)), completed_at: iso(exec0) }, + { name: "execute", started_at: iso(exec0), completed_at: iso(new Date(exec0.getTime() + opts.executeMs)) }, + ], + }, + ], + }) +} + +describe("dbt_cte_profile timing", () => { + let target: string + beforeEach(async () => { + target = await mkdtemp(path.join(os.tmpdir(), "dbt-cte-profile-target-")) + adapterTarget = target + }) + afterEach(async () => { + await rm(target, { recursive: true, force: true }) + }) + + const compiledOf = (sql: string) => sql.replace(/\{\{[^}]*\}\}/g, "x").trim() + const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms)) + + test("the time is dbt's own timing of the query, without dbt's start-up", async () => { + respond = async (sql) => { + const started = new Date() + // A command-line dbt spends most of the call starting up. + await sleep(60) + await writeFile(path.join(target, "run_results.json"), runResults({ started, executeMs: 7, compiled: compiledOf(sql) })) + return { columnNames: ["_PROFILE_COUNT"], data: [{ _PROFILE_COUNT: 42 }], compiledSql: `/* {"app": "dbt"} */\n${compiledOf(sql)}\n limit 1` } + } + const tool = await initTool(DbtCteProfileTool) + const result = await tool.execute({ model: "customers", cte: "orders" }, ctx) + expect(result.metadata).toMatchObject({ success: true, timing: "dbt", queryMs: 7, rows: 42 }) + expect(result.metadata.totalMs).toBeGreaterThanOrEqual(60) + expect(result.output).toContain("orders: 7 ms on the warehouse") + expect(result.output).toContain(`${result.metadata.totalMs} ms in all`) + expect(queries).toHaveLength(1) + expect(baselines).toEqual([]) + }) + + test("without dbt's timing, a trivial query's time through dbt is taken off, measured once per adapter", async () => { + respond = async (sql) => { + await sleep(sql === dbtCteProfileInternals.baselineQuery ? 40 : 100) + return { columnNames: ["_PROFILE_COUNT"], data: [{ _PROFILE_COUNT: 42 }] } + } + const tool = await initTool(DbtCteProfileTool) + const first = await tool.execute({ model: "customers", cte: "orders" }, ctx) + const second = await tool.execute({ model: "customers", cte: "final" }, ctx) + for (const result of [first, second]) { + expect(result.metadata.timing).toBe("baseline") + expect(result.metadata.queryMs).toBe(result.metadata.totalMs - result.metadata.startupMs) + expect(result.metadata.startupMs).toBeGreaterThanOrEqual(35) + expect(result.metadata.queryMs).toBeLessThan(result.metadata.totalMs) + } + expect(first.metadata.startupMs).toBe(second.metadata.startupMs) + expect(baselines).toEqual([{ sql: dbtCteProfileInternals.baselineQuery, name: "", limit: 1 }]) + }) + + test("a run_results.json from before the call or from another command is not this query's timing", async () => { + const before = new Date(Date.now() - 60_000) + await writeFile(path.join(target, "run_results.json"), runResults({ started: before, executeMs: 3, compiled: "select 1" })) + expect(dbtQueryMs(target, Date.now())).toBeUndefined() + const now = Date.now() + await writeFile(path.join(target, "run_results.json"), runResults({ started: new Date(now + 5), executeMs: 3, compiled: "select 1" })) + expect(dbtQueryMs(target, now, "select 2 limit 1")).toBeUndefined() + expect(dbtQueryMs(target, now, "/* c */ select 1 limit 1")).toBe(3) + await writeFile(path.join(target, "run_results.json"), runResults({ started: new Date(now + 5), executeMs: 3, compiled: "select 1", which: "build" })) + expect(dbtQueryMs(target, now)).toBeUndefined() + }) + + test("dbt's timestamps without a zone are read as UTC", async () => { + const now = Date.now() + const naive = runResults({ started: new Date(now + 5), executeMs: 9, compiled: "select 1" }).replaceAll(/(\d{3})123Z/g, "$1123") + await writeFile(path.join(target, "run_results.json"), naive) + expect(dbtQueryMs(target, now)).toBe(9) + }) + + test("when the trivial query fails too, the call's own time is given as including start-up, and the next call measures again", async () => { + let fail = true + respond = async (sql) => { + if (sql === dbtCteProfileInternals.baselineQuery && fail) throw new Error("no warehouse") + return { columnNames: ["_PROFILE_COUNT"], data: [{ _PROFILE_COUNT: 1 }] } + } + const tool = await initTool(DbtCteProfileTool) + const first = await tool.execute({ model: "customers", cte: "orders" }, ctx) + expect(first.metadata).toMatchObject({ timing: "wall" }) + expect(first.metadata.queryMs).toBe(first.metadata.totalMs) + expect(first.output).toContain("start-up included") + fail = false + const second = await tool.execute({ model: "customers", cte: "orders" }, ctx) + expect(second.metadata.timing).toBe("baseline") + expect(baselines).toHaveLength(2) + }) +}) diff --git a/packages/opencode/test/altimate/dbt-health.test.ts b/packages/opencode/test/altimate/dbt-health.test.ts index 5cf8e83106..8277e4dc2b 100644 --- a/packages/opencode/test/altimate/dbt-health.test.ts +++ b/packages/opencode/test/altimate/dbt-health.test.ts @@ -1,7 +1,7 @@ // dbt_health: the DataPilot run through the project's dbt adapter. The adapter and the Python run are // replaced through their seams, so these tests need no dbt, no Python and no DataPilot. import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, test } from "bun:test" -import { mkdir, mkdtemp, realpath, rm, symlink, utimes, writeFile } from "node:fs/promises" +import { mkdir, mkdtemp, readFile, realpath, rm, symlink, utimes, writeFile } from "node:fs/promises" import fs from "node:fs" import os from "node:os" import path from "node:path" @@ -198,6 +198,28 @@ describe("dbt_health", () => { ]) }) + test("reads the project's catalog when the adapter's own dbt target folder has none", async () => { + // The tools' dbt writes to a folder of its own under target/, where `dbt docs generate` never puts a catalog. + const own = path.join(project, "target", ".altimate-code") + await mkdir(own, { recursive: true }) + await writeFile(path.join(own, "manifest.json"), await readFile(path.join(project, "target", "manifest.json"), "utf8")) + await writeFile(path.join(project, "target", "catalog.json"), "{}") + dbtAdapterInternals.create = async () => ({ + isInitialized: () => true, + getProjectRoot: () => project, + getManifestPath: () => path.join(own, "manifest.json"), + parseManifest: async () => parsed, + dispose: async () => {}, + }) + try { + await run() + } finally { + await rm(path.join(project, "target", "catalog.json")) + await rm(own, { recursive: true, force: true }) + } + expect(python.calls[0].args).toEqual([path.join(own, "manifest.json"), path.join(project, "target", "catalog.json"), ""]) + }) + test("runs DataPilot under the Python the dbt config names, not one found for the project", async () => { dbtAdapterInternals.readConfig = async () => ({ projectRoot: project, pythonPath: "/pyenv/dbt/bin/python", dbtIntegration: "corecommand", queryLimit: 500 }) await run() diff --git a/packages/opencode/test/altimate/dbt-sql-graph.test.ts b/packages/opencode/test/altimate/dbt-sql-graph.test.ts index 58d6ae311e..e0ce0eaecf 100644 --- a/packages/opencode/test/altimate/dbt-sql-graph.test.ts +++ b/packages/opencode/test/altimate/dbt-sql-graph.test.ts @@ -15,6 +15,8 @@ import { dbtAdapterInternals, resetDbtAdapter } from "@/altimate/native/connecti import { initTool } from "./tool-fixture" import { buildSqlGraph, + clauseName, + CLAUSE_NAME_CHARS, formatSqlGraph, mergeColumns, remapSqlLineage, @@ -625,7 +627,7 @@ describe("the backend's real node shapes", () => { expect(byId["raw.customers_222"]).toMatchObject({ nodeId: "source.shop.raw.customers", type: "source" }) expect(byId.LEFT_JOIN_9).toEqual({ id: "LEFT_JOIN_9", - name: "LEFT_JOIN_9", + name: "LEFT JOIN raw.customers ON 1 = 1", type: "LEFT_JOIN", sql: "LEFT JOIN raw.customers ON 1 = 1", columns: [], @@ -633,6 +635,13 @@ describe("the backend's real node shapes", () => { expect(byId.WHERE_7.sql).toBe("WHERE amount > 0") }) + test("a join or filter is named by its clause, not the backend's generated key, which stays its id", () => { + const byId = Object.fromEntries(graph.nodes.map((n) => [n.id, n])) + expect(byId.WHERE_7.id).toBe("WHERE_7") + expect(byId.WHERE_7.name).toBe("WHERE amount > 0") + expect(graph.edges).toContainEqual(["WHERE_7", "orders"]) + }) + test("two relations of one name and schema in different databases are told apart by the database", () => { const twoDbs: SqlLineageResponse = { tableEdges: [], @@ -845,3 +854,37 @@ describe("final-select identity and the text output", () => { expect(text).toContain(" sql: select id from t") }) }) + +describe("clauseName", () => { + // The shapes the backend sends for a dbt model's joins, filters and groups (DuckDB, quoted relations). + test.each([ + [{ type: "FILTER", expression: "WHERE stg_orders.status <> 'returned'" }, "WHERE stg_orders.status <> 'returned'"], + [{ type: "GROUP", expression: "GROUP BY\n items.order_id" }, "GROUP BY items.order_id"], + [ + { type: "INNER_JOIN", expression: 'JOIN "shop"."main"."stg_products" AS products ON items.product_id = products.product_id' }, + "INNER JOIN stg_products AS products ON items.product_id = products.product_id", + ], + [{ type: "LEFT_JOIN", expression: "LEFT JOIN payments AS payments ON orders.order_id = payments.order_id" }, "LEFT JOIN payments ON orders.order_id = payments.order_id"], + [{ type: "INNER_JOIN", expression: "JOIN `proj`.`ds`.`orders` AS `orders` ON a = b" }, "INNER JOIN orders ON a = b"], + [{ type: "LEFT_JOIN", expression: "LEFT JOIN raw.customers ON 1 = 1" }, "LEFT JOIN raw.customers ON 1 = 1"], + ])("%o is named %p", (details, name) => { + expect(clauseName(details)).toBe(name) + }) + + test("an AS inside a condition is left as written", () => { + expect(clauseName({ type: "FILTER", expression: "WHERE CAST(a.b AS INT) > 0" })).toBe("WHERE CAST(a.b AS INT) > 0") + }) + + test("a long clause is cut with an ellipsis, and a node with no clause has no name of its own", () => { + const name = clauseName({ type: "FILTER", expression: `WHERE ${"x = 1 AND ".repeat(30)}y = 2` })! + expect(name.length).toBe(CLAUSE_NAME_CHARS) + expect(name.endsWith("…")).toBe(true) + expect(clauseName({ type: "GROUP" })).toBeUndefined() + const graph = remapSqlLineage( + { tableEdges: [], details: { Sort_1: { type: "SORT" } } }, + { name: "m", uniqueId: "model.p.m", resourceType: "model" }, + [], + ) + expect(graph.nodes.find((n) => n.id === "Sort_1")?.name).toBe("Sort_1") + }) +}) diff --git a/packages/opencode/test/altimate/impact-analysis.test.ts b/packages/opencode/test/altimate/impact-analysis.test.ts index 61845bbc15..a1638accd0 100644 --- a/packages/opencode/test/altimate/impact-analysis.test.ts +++ b/packages/opencode/test/altimate/impact-analysis.test.ts @@ -350,3 +350,29 @@ describe("impact_analysis: column reach carries how each column comes from the t } }) }) + +describe("impact_analysis: invalid arguments", () => { + // The CLI and MCP print this message as the answer: it has to say which field is wrong. + const message = async (args: unknown) => { + const tool = await initTool(ImpactAnalysisTool) + return Promise.resolve() + .then(() => tool.execute(args, ctx)) + .then( + () => "", + (error: unknown) => (error instanceof Error ? error.message : String(error)), + ) + } + + test("a missing change_type is named, with the values it takes", async () => { + const text = await message({ model: "stg_orders" }) + expect(text).toContain("invalid arguments: change_type is required (expected one of \"remove\"|\"rename\"|\"retype\"|\"add\"|\"modify\")") + expect(text).not.toContain("Declaration") + }) + + test("an unknown change_type and a wrong type are named field by field", async () => { + const text = await message({ model: 7, change_type: "drop" }) + expect(text).toContain("model: Invalid input: expected string, received number") + expect(text).toContain("change_type: Invalid option: expected one of") + expect(text).not.toContain("Declaration") + }) +}) diff --git a/packages/opencode/test/altimate/tool-cli/daemon.test.ts b/packages/opencode/test/altimate/tool-cli/daemon.test.ts index 00a4148ae8..d942edc716 100644 --- a/packages/opencode/test/altimate/tool-cli/daemon.test.ts +++ b/packages/opencode/test/altimate/tool-cli/daemon.test.ts @@ -140,6 +140,17 @@ describe("connection env in the socket name", () => { } }) + test("changes with the Python environment the dbt adapter runs dbt with", async () => { + const dir = await temp() + const none = socketName(dir, "1.0.0", undefined, undefined, { PATH: "/bin" }) + const venv = socketName(dir, "1.0.0", undefined, undefined, { PATH: "/bin", VIRTUAL_ENV: "/p/.venv" }) + // A daemon started outside the virtualenv found another Python: the caller inside it gets its own daemon. + expect(venv).not.toBe(none) + expect(socketName(dir, "1.0.0", undefined, undefined, { VIRTUAL_ENV: "/q/.venv" })).not.toBe(venv) + expect(socketName(dir, "1.0.0", undefined, undefined, { CONDA_PREFIX: "/opt/conda/envs/dbt" })).not.toBe(none) + expect(socketName(dir, "1.0.0", undefined, undefined, { VIRTUAL_ENV: "" })).toBe(none) + }) + test("uses the registry's prefix", async () => { const { CONN_ENV_PREFIX } = await import("../../../src/altimate/native/connections/registry") const dir = await temp() From 42667982ac283777692f05fabd1774d330d4c42a Mon Sep 17 00:00:00 2001 From: Harish Kumar Date: Fri, 20 Nov 2026 00:09:41 +0530 Subject: [PATCH 2/7] fix: keep Fusion and dbt Cloud on the project's target, tie a CTE's timing to its own run - The tools' dbt adapter writes to a target folder of its own only in the core and corecommand modes: Fusion and dbt Cloud read /target whatever dbt is told. - dbt_cte_profile tags each count query with a unique comment and takes dbt's timing only from a run_results.json whose compiled SQL carries that tag, so a concurrent count of the same CTE (or another daemon on the project) can't lend its time. - A call waiting on another call's start-up measurement stops at its own abort; the shared measurement isn't cancelled by the call that started it. - The new dbt-tools adapter test runs in the package's test command. Co-Authored-By: Claude Opus 5.5 --- docs/docs/usage/tool.md | 2 +- packages/dbt-tools/package.json | 2 +- .../altimate/native/connections/register.ts | 14 +++- .../src/altimate/tools/dbt-cte-profile.ts | 63 +++++++++----- .../test/altimate/dbt-adapter-access.test.ts | 16 ++++ .../test/altimate/dbt-cte-profile.test.ts | 82 ++++++++++++++++--- 6 files changed, 144 insertions(+), 35 deletions(-) diff --git a/docs/docs/usage/tool.md b/docs/docs/usage/tool.md index 59a4526e97..a21a9e8150 100644 --- a/docs/docs/usage/tool.md +++ b/docs/docs/usage/tool.md @@ -75,7 +75,7 @@ altimate-code tool daemon stop | `ALTIMATE_TOOL_DAEMON=0` | Always run in-process. | | `ALTIMATE_TOOL_DAEMON_IDLE_MS` | Idle time before the daemon exits (default `600000`). | -The tools that start the dbt adapter (`dbt_lineage`, `dbt_cte_profile`, `dbt_sql_graph`, `dbt_health`, `dbt_cost_estimate`, and `sql_execute` routed through dbt) have dbt write its artifacts to `/.altimate-code/` rather than the project's own `target/`: a dbt command rewrites `manifest.json` with compiled SQL for only what it compiled, so the project's `target/manifest.json` is left as your own `dbt compile` wrote it. +The tools that start the dbt adapter (`dbt_lineage`, `dbt_cte_profile`, `dbt_sql_graph`, `dbt_health`, `dbt_cost_estimate`, and `sql_execute` routed through dbt) have dbt write its artifacts to `/.altimate-code/` rather than the project's own `target/` in the `core` and `corecommand` integration modes: a dbt command rewrites `manifest.json` with compiled SQL for only what it compiled, so the project's `target/manifest.json` is left as your own `dbt compile` wrote it. The `fusion` and `cloud` modes read `/target` whatever dbt is told, so there dbt still writes to it. The daemon listens on a local socket (a named pipe on Windows) under the Altimate Code state directory and removes it on exit. diff --git a/packages/dbt-tools/package.json b/packages/dbt-tools/package.json index 66639efd2f..44174b14b5 100644 --- a/packages/dbt-tools/package.json +++ b/packages/dbt-tools/package.json @@ -9,7 +9,7 @@ "scripts": { "build": "bun build src/index.ts --outdir dist --target node --format esm && bun run script/copy-python.ts", "typecheck": "tsc --noEmit", - "test": "bun test test/cli.test.ts test/config.test.ts test/dbt-cli.test.ts test/dbt-resolve.test.ts test/adapter-bun.test.ts test/stale-manifest.test.ts test/build-integrity.test.ts --timeout 30000", + "test": "bun test test/cli.test.ts test/config.test.ts test/dbt-cli.test.ts test/dbt-resolve.test.ts test/adapter-bun.test.ts test/stale-manifest.test.ts test/build-integrity.test.ts test/adapter-target-path.test.ts --timeout 30000", "test:e2e": "bun test test/e2e/ --timeout 300000" }, "dependencies": { diff --git a/packages/opencode/src/altimate/native/connections/register.ts b/packages/opencode/src/altimate/native/connections/register.ts index 2f85c1cfb7..39fcf6139f 100644 --- a/packages/opencode/src/altimate/native/connections/register.ts +++ b/packages/opencode/src/altimate/native/connections/register.ts @@ -127,9 +127,12 @@ async function ensureDbtAdapter(): Promise { // Create the adapter. Its dbt commands write their artifacts to a folder of their own: the project's // target/manifest.json is the user's (and other tools'), and a tool's compile or show would rewrite // it with compiled SQL for only the nodes that command compiled. + // Only in the modes whose dbt-integration reads the artifacts back from where dbt wrote them. const create = dbtAdapterInternals.create ?? (await import("../../../../../dbt-tools/src/adapter")).create - const targetPath = await dbtToolTargetPath(dbtConfig.projectRoot) - return publish(await create(dbtConfig, { targetPath }), dbtConfig.projectRoot) + const options = SEPARATE_TARGET_MODES.has(dbtConfig.dbtIntegration || "core") + ? { targetPath: await dbtToolTargetPath(dbtConfig.projectRoot) } + : {} + return publish(await create(dbtConfig, options), dbtConfig.projectRoot) } catch { // dbt-tools not available or config invalid — fall back to native return publish(null) @@ -141,6 +144,13 @@ async function ensureDbtAdapter(): Promise { return mine } +/** + * The dbt-integration modes (dbt.json's `dbtIntegration`, `core` when unset) that read the manifest back from the + * target path dbt itself reports, DBT_TARGET_PATH included. Fusion and dbt Cloud read `/target` whatever + * dbt was told, so they keep writing there: a separate folder would leave them reading a manifest nothing updates. + */ +export const SEPARATE_TARGET_MODES: ReadonlySet = new Set(["core", "corecommand"]) + /** The folder, inside the project's target folder, where the tools' dbt adapter writes its dbt artifacts. */ export const DBT_TOOL_TARGET_DIR = ".altimate-code" diff --git a/packages/opencode/src/altimate/tools/dbt-cte-profile.ts b/packages/opencode/src/altimate/tools/dbt-cte-profile.ts index 3875acfacd..78dd75ed15 100644 --- a/packages/opencode/src/altimate/tools/dbt-cte-profile.ts +++ b/packages/opencode/src/altimate/tools/dbt-cte-profile.ts @@ -1,3 +1,4 @@ +import crypto from "crypto" import fs from "fs" import path from "path" import z from "zod" @@ -161,30 +162,48 @@ function utc(value: unknown): number { return Date.parse(/(Z|[+-]\d\d:?\d\d)$/i.test(value) ? value : `${value}Z`) } -/** The run_results.json dbt wrote for the count query, as the warehouse time of its execute step, in ms. */ -export function dbtQueryMs(targetDir: string, startedAt: number, compiledSql?: unknown): number | undefined { +/** + * The warehouse time, in ms, of the execute step of the count query that carries `tag`, from the + * run_results.json dbt wrote for it; undefined when that file is not this query's. + */ +export function dbtQueryMs(targetDir: string, startedAt: number, tag: string): number | undefined { let raw: any try { raw = JSON.parse(fs.readFileSync(path.join(targetDir, "run_results.json"), "utf-8")) } catch { return undefined } - // Only the file this call's command wrote: a later start than the call, a `show`, one inline node, - // and its compiled SQL inside what the adapter ran, when it says (another command may have run meanwhile). + // Only the file this call's command wrote: a start after the call's, a `show` of one inline node, and + // this call's tag in its compiled SQL. Other commands (another call's count of the same CTE, another + // daemon on the project) write the same file, and only the tag tells two runs of one query apart. const invoked = utc(raw?.metadata?.invocation_started_at ?? raw?.metadata?.generated_at) if (!Number.isFinite(invoked) || invoked < startedAt) return undefined if (raw?.args?.which !== undefined && raw.args.which !== "show") return undefined const results = Array.isArray(raw?.results) ? raw.results : [] if (results.length !== 1) return undefined const node = results[0] - // The adapter's SQL is what ran: the node's compiled code inside dbt's query comment and limit. - if (typeof compiledSql === "string" && typeof node?.compiled_code === "string" && !compiledSql.includes(node.compiled_code.trim())) return undefined + if (typeof node?.compiled_code !== "string" || !node.compiled_code.includes(tag)) return undefined const execute = Array.isArray(node?.timing) ? node.timing.find((t: any) => t?.name === "execute") : undefined const span = utc(execute?.completed_at) - utc(execute?.started_at) if (Number.isFinite(span) && span >= 0) return Math.round(span) return typeof node?.execution_time === "number" && node.execution_time >= 0 ? Math.round(node.execution_time * 1000) : undefined } +/** The comment a count query starts with, unique to the call, by which its run_results.json is known. */ +export function profileTag(): string { + return `altimate-code cte profile ${crypto.randomUUID()}` +} + +/** `work`, or the signal's reason as soon as it aborts. */ +function untilAbort(work: Promise, signal: AbortSignal): Promise { + signal.throwIfAborted() + return new Promise((resolve, reject) => { + const stop = () => reject(signal.reason) + signal.addEventListener("abort", stop, { once: true }) + work.then(resolve, reject).finally(() => signal.removeEventListener("abort", stop)) + }) +} + type QueryTime = { source: "dbt" | "baseline" | "wall"; queryMs: number; startupMs?: number } /** @@ -193,16 +212,18 @@ type QueryTime = { source: "dbt" | "baseline" | "wall"; queryMs: number; startup * run_results.json it wrote; else the call's time less a trivial query's time through the same adapter, * measured once per adapter; else the call's time as is. */ -async function queryTime(adapter: any, call: { startedAt: number; totalMs: number; compiledSql?: unknown }, ctx: Pick): Promise { +async function queryTime(adapter: any, call: { startedAt: number; totalMs: number; tag: string }, ctx: Pick): Promise { const dir = typeof adapter.getTargetPath === "function" ? adapter.getTargetPath() : undefined - const fromDbt = typeof dir === "string" && dir ? dbtQueryMs(dir, call.startedAt, call.compiledSql) : undefined + const fromDbt = typeof dir === "string" && dir ? dbtQueryMs(dir, call.startedAt, call.tag) : undefined if (fromDbt !== undefined) return { source: "dbt", queryMs: Math.min(fromDbt, call.totalMs) } let baseline = baselines.get(adapter) if (!baseline) { - baseline = (async () => { + // Shared by the calls that wait for it, so no one call's abort cancels it: each call stops waiting at its + // own abort, and the measurement is bounded by the query timeout. + const measured: Promise = (async () => { const started = performance.now() try { - await executeQuery({ type: "dbt" }, {}, { signal: ctx.abort, sessionID: ctx.sessionID }, () => + await executeQuery({ type: "dbt" }, {}, { sessionID: ctx.sessionID }, () => adapter.immediatelyExecuteSQLWithLimit(dbtCteProfileInternals.baselineQuery, "", 1), ) return Math.round(performance.now() - started) @@ -210,15 +231,15 @@ async function queryTime(adapter: any, call: { startedAt: number; totalMs: numbe return undefined } })() + // A failed measurement is not kept: the next call measures again. + void measured.then((ms) => { + if (ms === undefined && baselines.get(adapter) === measured) baselines.delete(adapter) + }) + baseline = measured baselines.set(adapter, baseline) } - const startupMs = await baseline - // A failed or cancelled measurement is not kept: the next call measures again. - if (startupMs === undefined) { - baselines.delete(adapter) - ctx.abort.throwIfAborted() - return { source: "wall", queryMs: call.totalMs } - } + const startupMs = await untilAbort(baseline, ctx.abort) + if (startupMs === undefined) return { source: "wall", queryMs: call.totalMs } const startup = Math.min(startupMs, call.totalMs) return { source: "baseline", queryMs: call.totalMs - startup, startupMs: startup } } @@ -330,12 +351,14 @@ export const DbtCteProfileTool = Tool.define("dbt_cte_profile", { const started = performance.now() try { // The adapter has no cancel of its own: the run's timeout, budget and abort end the wait, not the query. - const result: { data?: Record[]; compiledSql?: unknown } = await executeQuery({ type: "dbt" }, {}, { signal: ctx.abort, sessionID: ctx.sessionID }, () => - adapter.immediatelyExecuteSQLWithLimit(query, path.basename(modelPath, ".sql"), 1), + // The tag makes this run's compiled SQL its own, so its run_results.json can't be taken for another's. + const tag = profileTag() + const result: { data?: Record[] } = await executeQuery({ type: "dbt" }, {}, { signal: ctx.abort, sessionID: ctx.sessionID }, () => + adapter.immediatelyExecuteSQLWithLimit(`/* ${tag} */\n${query}`, path.basename(modelPath, ".sql"), 1), ) const totalMs = Math.round(performance.now() - started) const rows = extractRowCount(result?.data) - const timing = await queryTime(adapter, { startedAt, totalMs, compiledSql: result?.compiledSql }, ctx) + const timing = await queryTime(adapter, { startedAt, totalMs, tag }, ctx) const rowText = `${rows} row${rows === 1 ? "" : "s"}` const output = timing.source === "dbt" diff --git a/packages/opencode/test/altimate/dbt-adapter-access.test.ts b/packages/opencode/test/altimate/dbt-adapter-access.test.ts index 7611e873cf..3b94ec1b8e 100644 --- a/packages/opencode/test/altimate/dbt-adapter-access.test.ts +++ b/packages/opencode/test/altimate/dbt-adapter-access.test.ts @@ -203,6 +203,22 @@ describe("dbtAdapterFor", () => { expect(options).toEqual({ targetPath: path.join(project, "target", DBT_TOOL_TARGET_DIR) }) }) + test("Fusion and dbt Cloud, which read /target whatever dbt is told, keep dbt writing there", async () => { + const seen: ({ targetPath?: string } | undefined)[] = [] + dbtAdapterInternals.create = async (_config, given) => { + seen.push(given) + return { isInitialized: () => true, dispose: async () => {} } + } + for (const dbtIntegration of ["fusion", "cloud", "core", ""]) { + resetDbtAdapter() + dbtAdapterInternals.readConfig = async () => ({ projectRoot: project, pythonPath: "python3", dbtIntegration, queryLimit: 500 }) + await dbtAdapterFor(ctx) + } + const own = { targetPath: path.join(project, "target", DBT_TOOL_TARGET_DIR) } + // An unset mode is dbt-integration's default, core. + expect(seen).toEqual([{}, {}, own, own]) + }) + test("the tools' target folder follows the project's own target-path and DBT_TARGET_PATH", async () => { const yml = path.join(project, "dbt_project.yml") const saved = readFileSync(yml, "utf8") diff --git a/packages/opencode/test/altimate/dbt-cte-profile.test.ts b/packages/opencode/test/altimate/dbt-cte-profile.test.ts index b378368600..1547839fa6 100644 --- a/packages/opencode/test/altimate/dbt-cte-profile.test.ts +++ b/packages/opencode/test/altimate/dbt-cte-profile.test.ts @@ -553,7 +553,7 @@ describe("dbt_cte_profile timing", () => { // A command-line dbt spends most of the call starting up. await sleep(60) await writeFile(path.join(target, "run_results.json"), runResults({ started, executeMs: 7, compiled: compiledOf(sql) })) - return { columnNames: ["_PROFILE_COUNT"], data: [{ _PROFILE_COUNT: 42 }], compiledSql: `/* {"app": "dbt"} */\n${compiledOf(sql)}\n limit 1` } + return { columnNames: ["_PROFILE_COUNT"], data: [{ _PROFILE_COUNT: 42 }] } } const tool = await initTool(DbtCteProfileTool) const result = await tool.execute({ model: "customers", cte: "orders" }, ctx) @@ -562,6 +562,7 @@ describe("dbt_cte_profile timing", () => { expect(result.output).toContain("orders: 7 ms on the warehouse") expect(result.output).toContain(`${result.metadata.totalMs} ms in all`) expect(queries).toHaveLength(1) + expect(queries[0].sql).toMatch(/^\/\* altimate-code cte profile [0-9a-f-]{36} \*\/\n/) expect(baselines).toEqual([]) }) @@ -583,23 +584,82 @@ describe("dbt_cte_profile timing", () => { expect(baselines).toEqual([{ sql: dbtCteProfileInternals.baselineQuery, name: "", limit: 1 }]) }) - test("a run_results.json from before the call or from another command is not this query's timing", async () => { + test("a run_results.json from before the call, of another command, or without the call's tag is not this query's timing", async () => { + const tag = "altimate-code cte profile t-1" + const mine = `/* ${tag} */ select 1` const before = new Date(Date.now() - 60_000) - await writeFile(path.join(target, "run_results.json"), runResults({ started: before, executeMs: 3, compiled: "select 1" })) - expect(dbtQueryMs(target, Date.now())).toBeUndefined() + await writeFile(path.join(target, "run_results.json"), runResults({ started: before, executeMs: 3, compiled: mine })) + expect(dbtQueryMs(target, Date.now(), tag)).toBeUndefined() const now = Date.now() - await writeFile(path.join(target, "run_results.json"), runResults({ started: new Date(now + 5), executeMs: 3, compiled: "select 1" })) - expect(dbtQueryMs(target, now, "select 2 limit 1")).toBeUndefined() - expect(dbtQueryMs(target, now, "/* c */ select 1 limit 1")).toBe(3) - await writeFile(path.join(target, "run_results.json"), runResults({ started: new Date(now + 5), executeMs: 3, compiled: "select 1", which: "build" })) - expect(dbtQueryMs(target, now)).toBeUndefined() + await writeFile(path.join(target, "run_results.json"), runResults({ started: new Date(now + 5), executeMs: 3, compiled: "/* altimate-code cte profile t-2 */ select 1" })) + expect(dbtQueryMs(target, now, tag)).toBeUndefined() + await writeFile(path.join(target, "run_results.json"), runResults({ started: new Date(now + 5), executeMs: 3, compiled: mine, which: "build" })) + expect(dbtQueryMs(target, now, tag)).toBeUndefined() + await writeFile(path.join(target, "run_results.json"), runResults({ started: new Date(now + 5), executeMs: 3, compiled: mine })) + expect(dbtQueryMs(target, now, tag)).toBe(3) }) test("dbt's timestamps without a zone are read as UTC", async () => { const now = Date.now() - const naive = runResults({ started: new Date(now + 5), executeMs: 9, compiled: "select 1" }).replaceAll(/(\d{3})123Z/g, "$1123") + const naive = runResults({ started: new Date(now + 5), executeMs: 9, compiled: "/* t */ select 1" }).replaceAll(/(\d{3})123Z/g, "$1123") await writeFile(path.join(target, "run_results.json"), naive) - expect(dbtQueryMs(target, now)).toBe(9) + expect(dbtQueryMs(target, now, "t")).toBe(9) + }) + + test("two calls counting the same CTE at once each take only their own run's timing", async () => { + // A's dbt writes its run_results.json, then B's overwrites it before A's call returns. + let releaseA = () => {} + const aWritten = new Promise((resolve) => (releaseA = resolve)) + let bDone = () => {} + const bWritten = new Promise((resolve) => (bDone = resolve)) + let calls = 0 + respond = async (sql) => { + if (sql === dbtCteProfileInternals.baselineQuery) return { columnNames: ["N"], data: [{ N: 1 }] } + const which = calls++ === 0 ? "A" : "B" + const started = new Date() + await sleep(5) + if (which === "A") { + await writeFile(path.join(target, "run_results.json"), runResults({ started, executeMs: 500, compiled: compiledOf(sql) })) + releaseA() + await bWritten + } else { + await aWritten + await writeFile(path.join(target, "run_results.json"), runResults({ started, executeMs: 3, compiled: compiledOf(sql) })) + bDone() + } + return { columnNames: ["_PROFILE_COUNT"], data: [{ _PROFILE_COUNT: 42 }] } + } + const tool = await initTool(DbtCteProfileTool) + const [a, b] = await Promise.all([ + tool.execute({ model: "customers", cte: "orders" }, ctx), + tool.execute({ model: "customers", cte: "orders" }, ctx), + ]) + expect(b.metadata).toMatchObject({ timing: "dbt", queryMs: 3 }) + // B's file is not A's: A falls back to the start-up baseline rather than report B's 3 ms. + expect(a.metadata.timing).toBe("baseline") + // The same CTE, but each run's SQL is its own. + expect(queries[0].sql).not.toBe(queries[1].sql) + expect(queries[0].sql.replace(/^\/\* .*? \*\/\n/, "")).toBe(queries[1].sql.replace(/^\/\* .*? \*\/\n/, "")) + }) + + test("a call waiting on another call's start-up measurement stops at its own abort", async () => { + let finish = () => {} + const held = new Promise((resolve) => (finish = resolve)) + respond = async (sql) => { + if (sql === dbtCteProfileInternals.baselineQuery) await held + return { columnNames: ["_PROFILE_COUNT"], data: [{ _PROFILE_COUNT: 1 }] } + } + const tool = await initTool(DbtCteProfileTool) + const first = tool.execute({ model: "customers", cte: "orders" }, ctx) + while (baselines.length === 0) await sleep(1) + const controller = new AbortController() + const second = tool.execute({ model: "customers", cte: "final" }, { ...ctx, abort: controller.signal }) + await sleep(10) + controller.abort(new Error("stopped")) + await expect(second).rejects.toThrow("stopped") + finish() + expect((await first).metadata.timing).toBe("baseline") + expect(baselines).toHaveLength(1) }) test("when the trivial query fails too, the call's own time is given as including start-up, and the next call measures again", async () => { From 8eed76a3bc161a62989ed449f8917e83d9057e99 Mon Sep 17 00:00:00 2001 From: Harish Kumar Date: Fri, 20 Nov 2026 00:26:23 +0530 Subject: [PATCH 3/7] fix: review follow-ups for the dbt tool fixes - An adapter refusal under a saved ~/.altimate-code/dbt.json says to change its pythonPath, which wins over any virtualenv; the docs say the same. - A CTE's warehouse time comes only from dbt's execute step: a node's execution_time also counts the compile, so without the step the baseline is used. - Clause names keep whitespace inside quoted strings and identifiers, and drop a join's alias only when it is spelled exactly like the relation. - The baseline test no longer requires equal capped start-up times. Co-Authored-By: Claude Opus 5.5 --- docs/docs/usage/tool.md | 2 +- .../src/altimate/tools/dbt-adapter-access.ts | 24 +++++++++++++---- .../src/altimate/tools/dbt-cte-profile.ts | 4 +-- .../src/altimate/tools/dbt-sql-graph.ts | 27 +++++++++++++++++-- .../test/altimate/dbt-adapter-access.test.ts | 21 ++++++++++++++- .../test/altimate/dbt-cte-profile.test.ts | 6 ++++- .../test/altimate/dbt-sql-graph.test.ts | 5 ++++ 7 files changed, 77 insertions(+), 12 deletions(-) diff --git a/docs/docs/usage/tool.md b/docs/docs/usage/tool.md index a21a9e8150..ebeb53b063 100644 --- a/docs/docs/usage/tool.md +++ b/docs/docs/usage/tool.md @@ -68,7 +68,7 @@ altimate-code tool daemon status altimate-code tool daemon stop ``` -`status` and `stop` locate the running daemon the same way `run` does: by project directory, CLI version, the `--connections` file (if any), whether workspace tools are on, and the caller's `ALTIMATE_CODE_CONN_*`, `ALTIMATE_DISABLE_SCHEMA_VALUES`, `ALTIMATE_INTEGRATIONS`, `ALTIMATE_DISABLE_WORKSPACE`, `VIRTUAL_ENV` and `CONDA_PREFIX` environment variables. The last two pick the Python the daemon's dbt adapter runs dbt with, so activating the project's virtualenv gets a daemon that uses it rather than one started without it. Run them with the same flags and variables as the call that started the daemon, or they will report no daemon running for this project even though one is. +`status` and `stop` locate the running daemon the same way `run` does: by project directory, CLI version, the `--connections` file (if any), whether workspace tools are on, and the caller's `ALTIMATE_CODE_CONN_*`, `ALTIMATE_DISABLE_SCHEMA_VALUES`, `ALTIMATE_INTEGRATIONS`, `ALTIMATE_DISABLE_WORKSPACE`, `VIRTUAL_ENV` and `CONDA_PREFIX` environment variables. The last two are part of the daemon's identity because, when `~/.altimate-code/dbt.json` sets no `pythonPath` and the project has no `.venv`, `venv` or `env` folder of its own, they pick the Python the daemon's dbt adapter runs dbt with: activating the project's virtualenv then gets a daemon that uses it rather than one started without it. A `pythonPath` in `dbt.json` always wins; change it there. Run them with the same flags and variables as the call that started the daemon, or they will report no daemon running for this project even though one is. | Setting | Description | |---|---| diff --git a/packages/opencode/src/altimate/tools/dbt-adapter-access.ts b/packages/opencode/src/altimate/tools/dbt-adapter-access.ts index a808ae8845..9ea0525724 100644 --- a/packages/opencode/src/altimate/tools/dbt-adapter-access.ts +++ b/packages/opencode/src/altimate/tools/dbt-adapter-access.ts @@ -71,7 +71,7 @@ export async function dbtAdapterFor(ctx: Pick { + try { + const { path: configPath } = await import("../../../../dbt-tools/src/config") + return fs.existsSync(configPath()) + } catch { + return false + } } /** `p` with its links resolved, or as given when it can't be. */ diff --git a/packages/opencode/src/altimate/tools/dbt-cte-profile.ts b/packages/opencode/src/altimate/tools/dbt-cte-profile.ts index 78dd75ed15..6b13712394 100644 --- a/packages/opencode/src/altimate/tools/dbt-cte-profile.ts +++ b/packages/opencode/src/altimate/tools/dbt-cte-profile.ts @@ -185,8 +185,8 @@ export function dbtQueryMs(targetDir: string, startedAt: number, tag: string): n if (typeof node?.compiled_code !== "string" || !node.compiled_code.includes(tag)) return undefined const execute = Array.isArray(node?.timing) ? node.timing.find((t: any) => t?.name === "execute") : undefined const span = utc(execute?.completed_at) - utc(execute?.started_at) - if (Number.isFinite(span) && span >= 0) return Math.round(span) - return typeof node?.execution_time === "number" && node.execution_time >= 0 ? Math.round(node.execution_time * 1000) : undefined + // Only the execute step: the node's execution_time also counts compiling it. + return Number.isFinite(span) && span >= 0 ? Math.round(span) : undefined } /** The comment a count query starts with, unique to the call, by which its run_results.json is known. */ diff --git a/packages/opencode/src/altimate/tools/dbt-sql-graph.ts b/packages/opencode/src/altimate/tools/dbt-sql-graph.ts index 7db2a6edb0..b685391aa8 100644 --- a/packages/opencode/src/altimate/tools/dbt-sql-graph.ts +++ b/packages/opencode/src/altimate/tools/dbt-sql-graph.ts @@ -132,19 +132,42 @@ const IDENT = String.raw`(?:"(?:[^"]|"")+"|\`(?:[^\`]|\`\`)+\`|\[[^\]]+\]|[A-Za- // A join's own relation, right after its JOIN keyword: `JOIN "db"."schema"."table" AS alias`. const JOINED_RELATION = new RegExp(String.raw`^((?:[A-Za-z]+\s+)*JOIN\s+)(${IDENT}(?:\s*\.\s*${IDENT})*)(?:\s+AS)?\s+(?!ON\b|USING\b)(${IDENT})(?=\s|$)`, "i") +/** SQL on one line: runs of whitespace become one space, except inside quoted strings and identifiers. */ +function oneLine(sql: string): string { + let out = "" + for (let i = 0; i < sql.length; ) { + const ch = sql[i] + if (ch === "'" || ch === '"' || ch === "`") { + // To the closing quote; a doubled quote is an escaped one. + let j = i + 1 + while (j < sql.length && !(sql[j] === ch && sql[j + 1] !== ch)) j += sql[j] === ch ? 2 : 1 + out += sql.slice(i, j + 1) + i = j + 1 + } else if (/\s/.test(ch)) { + while (i < sql.length && /\s/.test(sql[i])) i++ + out += " " + } else { + out += ch + i++ + } + } + return out.trim() +} + /** * A name for a node the backend leaves unnamed (a join, filter, group or sort): its clause on one line, a relation * written as its own name (`LEFT JOIN payments ON …`, `WHERE status <> 'returned'`), cut at CLAUSE_NAME_CHARS. * Undefined when the node has no clause to show. */ export function clauseName(details: Pick): string | undefined { - let text = (details.expression ?? details.sql ?? "").replace(/\s+/g, " ").trim() + let text = oneLine(details.expression ?? details.sql ?? "") if (!text) return undefined // A joined relation keeps its last name part; its alias goes when it only repeats that name. Nothing else in // the clause is rewritten: an `AS` in a condition (a CAST) is not a relation. text = text.replace(JOINED_RELATION, (_all, join: string, relation: string, alias: string) => { const last = unquote(relation.match(new RegExp(IDENT, "g"))!.at(-1)!) - return `${join}${last.toLowerCase() === unquote(alias).toLowerCase() ? last : `${last} AS ${unquote(alias)}`}` + // Exactly the same spelling: a quoted alias that differs in case can name another relation. + return `${join}${last === unquote(alias) ? last : `${last} AS ${unquote(alias)}`}` }) // The backend writes an inner join as a bare JOIN; its type says which join it is. const kind = /^([A-Z]+)_JOIN$/i.exec(details.type ?? "")?.[1] diff --git a/packages/opencode/test/altimate/dbt-adapter-access.test.ts b/packages/opencode/test/altimate/dbt-adapter-access.test.ts index 3b94ec1b8e..dfb6462df3 100644 --- a/packages/opencode/test/altimate/dbt-adapter-access.test.ts +++ b/packages/opencode/test/altimate/dbt-adapter-access.test.ts @@ -2,7 +2,7 @@ // one gate the new dbt-adapter-backed tools go through, and the API helper for new endpoint files. The // dbt adapter is replaced through `dbtAdapterInternals`, so these tests need no Python or dbt. import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, test } from "bun:test" -import { chmod, mkdtemp, rm, writeFile } from "node:fs/promises" +import { chmod, mkdir, mkdtemp, rm, writeFile } from "node:fs/promises" import { existsSync, readFileSync, realpathSync } from "node:fs" import os from "node:os" import path from "node:path" @@ -193,6 +193,25 @@ describe("dbtAdapterFor", () => { expect(access.reason).toContain("altimate-code tool daemon stop") }) + test("with a saved dbt config, the refusal says to change its pythonPath, which wins over any virtualenv", async () => { + const home = await mkdtemp(path.join(os.tmpdir(), "dbt-adapter-access-home-")) + const saved = process.env.HOME + try { + await mkdir(path.join(home, ".altimate-code"), { recursive: true }) + await writeFile(path.join(home, ".altimate-code", "dbt.json"), "{}") + process.env.HOME = home + dbtAdapterInternals.create = async () => ({ isInitialized: () => false, dispose: async () => {} }) + const access = await dbtAdapterFor(ctx) + expect(access.reason).toContain("Python that ~/.altimate-code/dbt.json names (python3)") + expect(access.reason).toContain("Set pythonPath there") + expect(access.reason).not.toContain("Activate the project's virtualenv") + } finally { + if (saved === undefined) delete process.env.HOME + else process.env.HOME = saved + await rm(home, { recursive: true, force: true }) + } + }) + test("the tools' adapter is created with a dbt target folder of its own, inside the project's target", async () => { let options: { targetPath?: string } | undefined dbtAdapterInternals.create = async (_config, given) => { diff --git a/packages/opencode/test/altimate/dbt-cte-profile.test.ts b/packages/opencode/test/altimate/dbt-cte-profile.test.ts index 1547839fa6..51ec1e15e3 100644 --- a/packages/opencode/test/altimate/dbt-cte-profile.test.ts +++ b/packages/opencode/test/altimate/dbt-cte-profile.test.ts @@ -580,7 +580,6 @@ describe("dbt_cte_profile timing", () => { expect(result.metadata.startupMs).toBeGreaterThanOrEqual(35) expect(result.metadata.queryMs).toBeLessThan(result.metadata.totalMs) } - expect(first.metadata.startupMs).toBe(second.metadata.startupMs) expect(baselines).toEqual([{ sql: dbtCteProfileInternals.baselineQuery, name: "", limit: 1 }]) }) @@ -597,6 +596,11 @@ describe("dbt_cte_profile timing", () => { expect(dbtQueryMs(target, now, tag)).toBeUndefined() await writeFile(path.join(target, "run_results.json"), runResults({ started: new Date(now + 5), executeMs: 3, compiled: mine })) expect(dbtQueryMs(target, now, tag)).toBe(3) + // Without the execute step's timing there is no warehouse time: execution_time also counts the compile. + const noTiming = JSON.parse(runResults({ started: new Date(now + 5), executeMs: 3, compiled: mine })) + noTiming.results[0].timing = [] + await writeFile(path.join(target, "run_results.json"), JSON.stringify(noTiming)) + expect(dbtQueryMs(target, now, tag)).toBeUndefined() }) test("dbt's timestamps without a zone are read as UTC", async () => { diff --git a/packages/opencode/test/altimate/dbt-sql-graph.test.ts b/packages/opencode/test/altimate/dbt-sql-graph.test.ts index e0ce0eaecf..bde6bea773 100644 --- a/packages/opencode/test/altimate/dbt-sql-graph.test.ts +++ b/packages/opencode/test/altimate/dbt-sql-graph.test.ts @@ -871,6 +871,11 @@ describe("clauseName", () => { expect(clauseName(details)).toBe(name) }) + test("whitespace inside a quoted string or identifier is kept, and an alias differing only in case stays", () => { + expect(clauseName({ type: "FILTER", expression: "WHERE code = 'a, b''s'\n AND \"x y\" = 1" })).toBe("WHERE code = 'a, b''s' AND \"x y\" = 1") + expect(clauseName({ type: "LEFT_JOIN", expression: 'LEFT JOIN "db"."s"."Orders" AS "orders" ON 1 = 1' })).toBe("LEFT JOIN Orders AS orders ON 1 = 1") + }) + test("an AS inside a condition is left as written", () => { expect(clauseName({ type: "FILTER", expression: "WHERE CAST(a.b AS INT) > 0" })).toBe("WHERE CAST(a.b AS INT) > 0") }) From d46f446ffa33d5ca00eef92ca543ccd60de4ee13 Mon Sep 17 00:00:00 2001 From: Harish Kumar Date: Fri, 20 Nov 2026 00:30:22 +0530 Subject: [PATCH 4/7] fix: second round of review follow-ups for the dbt tool fixes - A finished CTE count waits at most 30 s for a stalled start-up measurement, then answers with its own time; the wait reuses dbt_cost_estimate's untilAbort. - A joined relation or alias that needs its quotes keeps them in the clause name. - Tests: the concurrent-count assertion no longer depends on call order, the abort test aborts only once the second call waits on the measurement, and the target-path tests clear and restore an inherited DBT_TARGET_PATH. Co-Authored-By: Claude Opus 5.5 --- .../test/adapter-target-path.test.ts | 3 +- .../src/altimate/tools/dbt-cte-profile.ts | 22 +++++++------- .../src/altimate/tools/dbt-sql-graph.ts | 6 ++-- .../test/altimate/dbt-adapter-access.test.ts | 8 +++++ .../test/altimate/dbt-cte-profile.test.ts | 29 ++++++++++++++++--- .../test/altimate/dbt-sql-graph.test.ts | 2 ++ 6 files changed, 53 insertions(+), 17 deletions(-) diff --git a/packages/dbt-tools/test/adapter-target-path.test.ts b/packages/dbt-tools/test/adapter-target-path.test.ts index 86957353a2..44fa984b73 100644 --- a/packages/dbt-tools/test/adapter-target-path.test.ts +++ b/packages/dbt-tools/test/adapter-target-path.test.ts @@ -47,7 +47,8 @@ describe.skipIf(process.platform === "win32")("create's target folder", () => { expect(process.env.DBT_TARGET_PATH).toBeUndefined() return seen() } finally { - if (saved !== undefined) process.env.DBT_TARGET_PATH = saved + if (saved === undefined) delete process.env.DBT_TARGET_PATH + else process.env.DBT_TARGET_PATH = saved rmSync(root, { recursive: true, force: true }) } } diff --git a/packages/opencode/src/altimate/tools/dbt-cte-profile.ts b/packages/opencode/src/altimate/tools/dbt-cte-profile.ts index 6b13712394..9b5a60af1d 100644 --- a/packages/opencode/src/altimate/tools/dbt-cte-profile.ts +++ b/packages/opencode/src/altimate/tools/dbt-cte-profile.ts @@ -8,6 +8,7 @@ import { projectRoot } from "../native/connections/registry" import { isQueryTimeoutError } from "@altimateai/drivers/types" import { executeQuery } from "../native/query-execution" import { dbtAdapterFor } from "./dbt-adapter-access" +import { untilAbort } from "./dbt-cost-estimate" import { buildCountQuery, detectCtes, extractRowCount, stripColumnList, unclosedJinjaBlock, unreadCteAfter, type CteInfo } from "./cte-detect" /** Folders a model search never enters. */ @@ -151,6 +152,8 @@ function nameMatches(list: CteInfo[], typed: string): CteInfo[] { export const dbtCteProfileInternals = { /** The trivial query whose time through dbt stands for dbt's own start-up, when dbt reports no timing. */ baselineQuery: "select 1 as _profile_baseline", + /** How long a finished count waits for that measurement before it answers with its own time as is. */ + baselineWaitMs: 30_000, } /** Per adapter: how long a trivial query takes through it, dbt's start-up with next to no warehouse work. */ @@ -194,14 +197,11 @@ export function profileTag(): string { return `altimate-code cte profile ${crypto.randomUUID()}` } -/** `work`, or the signal's reason as soon as it aborts. */ -function untilAbort(work: Promise, signal: AbortSignal): Promise { - signal.throwIfAborted() - return new Promise((resolve, reject) => { - const stop = () => reject(signal.reason) - signal.addEventListener("abort", stop, { once: true }) - work.then(resolve, reject).finally(() => signal.removeEventListener("abort", stop)) - }) +/** `work`'s value, or undefined once `ms` have passed first; `work` goes on either way. */ +function within(work: Promise, ms: number): Promise { + let timer: ReturnType | undefined + const late = new Promise((resolve) => (timer = setTimeout(() => resolve(undefined), ms))) + return Promise.race([work, late]).finally(() => clearTimeout(timer)) } type QueryTime = { source: "dbt" | "baseline" | "wall"; queryMs: number; startupMs?: number } @@ -238,7 +238,9 @@ async function queryTime(adapter: any, call: { startedAt: number; totalMs: numbe baseline = measured baselines.set(adapter, baseline) } - const startupMs = await untilAbort(baseline, ctx.abort) + // A measurement that stalls doesn't hold a count that has finished: past the wait, the count's own time is given. + const pending = baseline + const startupMs = await untilAbort(() => within(pending, dbtCteProfileInternals.baselineWaitMs), ctx.abort) if (startupMs === undefined) return { source: "wall", queryMs: call.totalMs } const startup = Math.min(startupMs, call.totalMs) return { source: "baseline", queryMs: call.totalMs - startup, startupMs: startup } @@ -250,7 +252,7 @@ function failure(title: string, output: string, extra: Record = export const DbtCteProfileTool = Tool.define("dbt_cte_profile", { description: - "List a dbt model's CTEs, or time one CTE with a row-count query. Without `cte` or `position` it returns the CTE names in source order (name, line, position) and runs nothing, so a read-only call works. With one of them it runs `SELECT COUNT(*)` over that CTE and the CTEs before it in the same WITH clause on the warehouse, and returns the row count and the query's milliseconds on the warehouse (`queryMs`): dbt's own timing of the query's execute step where dbt reports one (run_results.json), else the call's time less that of a trivial query through dbt, so a command-line dbt's start-up of a second or more per query is not counted; `totalMs` is the whole call, start-up included, and `timing` says which measure `queryMs` is (dbt, baseline, or wall when neither was available); that run needs write permission (`--allow-write`) because it starts the dbt adapter and compiles the model. Each run recomputes the earlier CTEs, so a model of N CTEs costs N queries. Only the model's own top-level WITH clause is read: a WITH inside a CTE body is part of that CTE. The list stops at a CTE whose name Jinja builds, and `incomplete` is set in the metadata then; a CTE whose count query reads is_incremental() or this, in its own SQL, an earlier CTE or the model header, is not run, because the count runs outside the model. Macros the query calls are not opened: one that reads is_incremental() inside is not caught, so treat the count of such a model as a full-history count.", + "List a dbt model's CTEs, or time one CTE with a row-count query. Without `cte` or `position` it returns the CTE names in source order (name, line, position) and runs nothing, so a read-only call works. With one of them it runs `SELECT COUNT(*)` over that CTE and the CTEs before it in the same WITH clause on the warehouse, and returns the row count and the query's milliseconds on the warehouse (`queryMs`): dbt's own timing of the query's execute step where dbt reports one (run_results.json), else the call's time less that of a trivial query through dbt, so a command-line dbt's start-up of a second or more per query is not counted; `totalMs` is the count query's whole call through dbt, start-up included, and `timing` says which measure `queryMs` is (dbt, baseline, or wall when neither was available); that run needs write permission (`--allow-write`) because it starts the dbt adapter and compiles the model. Each run recomputes the earlier CTEs, so a model of N CTEs costs N queries. Only the model's own top-level WITH clause is read: a WITH inside a CTE body is part of that CTE. The list stops at a CTE whose name Jinja builds, and `incomplete` is set in the metadata then; a CTE whose count query reads is_incremental() or this, in its own SQL, an earlier CTE or the model header, is not run, because the count runs outside the model. Macros the query calls are not opened: one that reads is_incremental() inside is not caught, so treat the count of such a model as a full-history count.", parameters: z.object({ model: z .string() diff --git a/packages/opencode/src/altimate/tools/dbt-sql-graph.ts b/packages/opencode/src/altimate/tools/dbt-sql-graph.ts index b685391aa8..a12569bbd9 100644 --- a/packages/opencode/src/altimate/tools/dbt-sql-graph.ts +++ b/packages/opencode/src/altimate/tools/dbt-sql-graph.ts @@ -165,9 +165,11 @@ export function clauseName(details: Pick { - const last = unquote(relation.match(new RegExp(IDENT, "g"))!.at(-1)!) + // A name that needs its quotes (a space, a keyword-like spelling) keeps them; a plain one is shown bare. + const shown = (part: string) => (/^[A-Za-z_][\w$]*$/.test(unquote(part)) ? unquote(part) : part) + const last = relation.match(new RegExp(IDENT, "g"))!.at(-1)! // Exactly the same spelling: a quoted alias that differs in case can name another relation. - return `${join}${last === unquote(alias) ? last : `${last} AS ${unquote(alias)}`}` + return `${join}${unquote(last) === unquote(alias) ? shown(last) : `${shown(last)} AS ${shown(alias)}`}` }) // The backend writes an inner join as a bare JOIN; its type says which join it is. const kind = /^([A-Z]+)_JOIN$/i.exec(details.type ?? "")?.[1] diff --git a/packages/opencode/test/altimate/dbt-adapter-access.test.ts b/packages/opencode/test/altimate/dbt-adapter-access.test.ts index dfb6462df3..3f353645b9 100644 --- a/packages/opencode/test/altimate/dbt-adapter-access.test.ts +++ b/packages/opencode/test/altimate/dbt-adapter-access.test.ts @@ -135,7 +135,11 @@ describe("feature tools in the allow-list", () => { describe("dbtAdapterFor", () => { let created: number + // The default target folder is the project's own `target`: an inherited DBT_TARGET_PATH would move it. + let inheritedTargetPath: string | undefined beforeEach(() => { + inheritedTargetPath = process.env.DBT_TARGET_PATH + delete process.env.DBT_TARGET_PATH created = 0 useConfig() dbtAdapterInternals.create = async () => { @@ -143,6 +147,10 @@ describe("dbtAdapterFor", () => { return { isInitialized: () => true, dispose: async () => {} } } }) + afterEach(() => { + if (inheritedTargetPath === undefined) delete process.env.DBT_TARGET_PATH + else process.env.DBT_TARGET_PATH = inheritedTargetPath + }) test("refuses a read-only call and never creates the adapter", async () => { const access = await dbtAdapterFor({ ...ctx, extra: { forceReadOnly: true } }) diff --git a/packages/opencode/test/altimate/dbt-cte-profile.test.ts b/packages/opencode/test/altimate/dbt-cte-profile.test.ts index 51ec1e15e3..6e747dbc51 100644 --- a/packages/opencode/test/altimate/dbt-cte-profile.test.ts +++ b/packages/opencode/test/altimate/dbt-cte-profile.test.ts @@ -638,9 +638,10 @@ describe("dbt_cte_profile timing", () => { tool.execute({ model: "customers", cte: "orders" }, ctx), tool.execute({ model: "customers", cte: "orders" }, ctx), ]) - expect(b.metadata).toMatchObject({ timing: "dbt", queryMs: 3 }) - // B's file is not A's: A falls back to the start-up baseline rather than report B's 3 ms. - expect(a.metadata.timing).toBe("baseline") + // Whichever call ran first wrote 500 ms and saw it overwritten: it falls back to the start-up baseline + // rather than report the other run's 3 ms, and only the run that wrote 3 ms reports it. + const timings = [a.metadata, b.metadata].map((m) => `${m.timing}:${m.timing === "dbt" ? m.queryMs : "-"}`).sort() + expect(timings).toEqual(["baseline:-", "dbt:3"]) // The same CTE, but each run's SQL is its own. expect(queries[0].sql).not.toBe(queries[1].sql) expect(queries[0].sql.replace(/^\/\* .*? \*\/\n/, "")).toBe(queries[1].sql.replace(/^\/\* .*? \*\/\n/, "")) @@ -658,7 +659,10 @@ describe("dbt_cte_profile timing", () => { while (baselines.length === 0) await sleep(1) const controller = new AbortController() const second = tool.execute({ model: "customers", cte: "final" }, { ...ctx, abort: controller.signal }) - await sleep(10) + // The second call's count has answered: it is now waiting on the first call's measurement. + while (queries.length < 2) await sleep(1) + await sleep(5) + expect(baselines).toHaveLength(1) controller.abort(new Error("stopped")) await expect(second).rejects.toThrow("stopped") finish() @@ -666,6 +670,23 @@ describe("dbt_cte_profile timing", () => { expect(baselines).toHaveLength(1) }) + test("a finished count waits only so long for a stalled start-up measurement", async () => { + respond = async (sql) => { + if (sql === dbtCteProfileInternals.baselineQuery) return new Promise(() => {}) + return { columnNames: ["_PROFILE_COUNT"], data: [{ _PROFILE_COUNT: 1 }] } + } + const saved = dbtCteProfileInternals.baselineWaitMs + dbtCteProfileInternals.baselineWaitMs = 20 + try { + const tool = await initTool(DbtCteProfileTool) + const result = await tool.execute({ model: "customers", cte: "orders" }, ctx) + expect(result.metadata.timing).toBe("wall") + expect(result.metadata.queryMs).toBe(result.metadata.totalMs) + } finally { + dbtCteProfileInternals.baselineWaitMs = saved + } + }) + test("when the trivial query fails too, the call's own time is given as including start-up, and the next call measures again", async () => { let fail = true respond = async (sql) => { diff --git a/packages/opencode/test/altimate/dbt-sql-graph.test.ts b/packages/opencode/test/altimate/dbt-sql-graph.test.ts index bde6bea773..734e4df81a 100644 --- a/packages/opencode/test/altimate/dbt-sql-graph.test.ts +++ b/packages/opencode/test/altimate/dbt-sql-graph.test.ts @@ -874,6 +874,8 @@ describe("clauseName", () => { test("whitespace inside a quoted string or identifier is kept, and an alias differing only in case stays", () => { expect(clauseName({ type: "FILTER", expression: "WHERE code = 'a, b''s'\n AND \"x y\" = 1" })).toBe("WHERE code = 'a, b''s' AND \"x y\" = 1") expect(clauseName({ type: "LEFT_JOIN", expression: 'LEFT JOIN "db"."s"."Orders" AS "orders" ON 1 = 1' })).toBe("LEFT JOIN Orders AS orders ON 1 = 1") + // A name that needs its quotes keeps them. + expect(clauseName({ type: "LEFT_JOIN", expression: 'LEFT JOIN "db"."s"."my table" AS "x y" ON 1 = 1' })).toBe('LEFT JOIN "my table" AS "x y" ON 1 = 1') }) test("an AS inside a condition is left as written", () => { From 818468702d95ae41414e8903b06d23b4de7b66da Mon Sep 17 00:00:00 2001 From: Harish Kumar Date: Fri, 20 Nov 2026 00:31:43 +0530 Subject: [PATCH 5/7] fix: clause names keep the whitespace inside a bracketed identifier Co-Authored-By: Claude Opus 5.5 --- packages/opencode/src/altimate/tools/dbt-sql-graph.ts | 9 +++++---- packages/opencode/test/altimate/dbt-sql-graph.test.ts | 1 + 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/packages/opencode/src/altimate/tools/dbt-sql-graph.ts b/packages/opencode/src/altimate/tools/dbt-sql-graph.ts index a12569bbd9..f90581e7c8 100644 --- a/packages/opencode/src/altimate/tools/dbt-sql-graph.ts +++ b/packages/opencode/src/altimate/tools/dbt-sql-graph.ts @@ -132,15 +132,16 @@ const IDENT = String.raw`(?:"(?:[^"]|"")+"|\`(?:[^\`]|\`\`)+\`|\[[^\]]+\]|[A-Za- // A join's own relation, right after its JOIN keyword: `JOIN "db"."schema"."table" AS alias`. const JOINED_RELATION = new RegExp(String.raw`^((?:[A-Za-z]+\s+)*JOIN\s+)(${IDENT}(?:\s*\.\s*${IDENT})*)(?:\s+AS)?\s+(?!ON\b|USING\b)(${IDENT})(?=\s|$)`, "i") -/** SQL on one line: runs of whitespace become one space, except inside quoted strings and identifiers. */ +/** SQL on one line: runs of whitespace become one space, except inside quoted strings and identifiers (`[…]` included). */ function oneLine(sql: string): string { let out = "" for (let i = 0; i < sql.length; ) { const ch = sql[i] - if (ch === "'" || ch === '"' || ch === "`") { - // To the closing quote; a doubled quote is an escaped one. + if (ch === "'" || ch === '"' || ch === "`" || ch === "[") { + // To the closing quote (`]` for a bracketed name); a doubled one is an escaped one. + const close = ch === "[" ? "]" : ch let j = i + 1 - while (j < sql.length && !(sql[j] === ch && sql[j + 1] !== ch)) j += sql[j] === ch ? 2 : 1 + while (j < sql.length && !(sql[j] === close && sql[j + 1] !== close)) j += sql[j] === close ? 2 : 1 out += sql.slice(i, j + 1) i = j + 1 } else if (/\s/.test(ch)) { diff --git a/packages/opencode/test/altimate/dbt-sql-graph.test.ts b/packages/opencode/test/altimate/dbt-sql-graph.test.ts index 734e4df81a..234b2da71e 100644 --- a/packages/opencode/test/altimate/dbt-sql-graph.test.ts +++ b/packages/opencode/test/altimate/dbt-sql-graph.test.ts @@ -874,6 +874,7 @@ describe("clauseName", () => { test("whitespace inside a quoted string or identifier is kept, and an alias differing only in case stays", () => { expect(clauseName({ type: "FILTER", expression: "WHERE code = 'a, b''s'\n AND \"x y\" = 1" })).toBe("WHERE code = 'a, b''s' AND \"x y\" = 1") expect(clauseName({ type: "LEFT_JOIN", expression: 'LEFT JOIN "db"."s"."Orders" AS "orders" ON 1 = 1' })).toBe("LEFT JOIN Orders AS orders ON 1 = 1") + expect(clauseName({ type: "FILTER", expression: "WHERE [order status]]x] = 1" })).toBe("WHERE [order status]]x] = 1") // A name that needs its quotes keeps them. expect(clauseName({ type: "LEFT_JOIN", expression: 'LEFT JOIN "db"."s"."my table" AS "x y" ON 1 = 1' })).toBe('LEFT JOIN "my table" AS "x y" ON 1 = 1') }) From 697a43a3dad9e3bcabc29feb5aa3295e51685b60 Mon Sep 17 00:00:00 2001 From: Harish Kumar Date: Fri, 20 Nov 2026 00:39:12 +0530 Subject: [PATCH 6/7] fix: the refusal blames dbt.json only when it names a pythonPath, and clause names stay on one line A dbt.json without a pythonPath, or one that doesn't parse, leaves the Python to discovery, so the refusal gives the virtualenv advice then. A line break inside a quoted literal becomes a space in a clause name. Co-Authored-By: Claude Opus 5.5 --- .../src/altimate/tools/dbt-adapter-access.ts | 8 ++++++-- .../opencode/src/altimate/tools/dbt-sql-graph.ts | 5 +++-- .../test/altimate/dbt-adapter-access.test.ts | 12 +++++++++++- .../opencode/test/altimate/dbt-sql-graph.test.ts | 1 + 4 files changed, 21 insertions(+), 5 deletions(-) diff --git a/packages/opencode/src/altimate/tools/dbt-adapter-access.ts b/packages/opencode/src/altimate/tools/dbt-adapter-access.ts index 9ea0525724..6beef71403 100644 --- a/packages/opencode/src/altimate/tools/dbt-adapter-access.ts +++ b/packages/opencode/src/altimate/tools/dbt-adapter-access.ts @@ -108,11 +108,15 @@ function pythonHint(python: string | undefined, saved: boolean): string { : `: dbt runs with the Python found when the tools started${which}, which may not be the project's. Activate the project's virtualenv (or set pythonPath in ~/.altimate-code/dbt.json), ${restart}` } -/** Whether ~/.altimate-code/dbt.json exists: its pythonPath, when set, is the Python dbt runs with. */ +/** + * Whether ~/.altimate-code/dbt.json picks the Python: it parses and names a pythonPath. A file without one, + * or one that doesn't parse, leaves the choice to discovery (project venv, VIRTUAL_ENV, CONDA_PREFIX, PATH). + */ async function savedDbtConfig(): Promise { try { const { path: configPath } = await import("../../../../dbt-tools/src/config") - return fs.existsSync(configPath()) + const saved = JSON.parse(await fs.promises.readFile(configPath(), "utf8")) as { pythonPath?: unknown } | null + return typeof saved?.pythonPath === "string" && saved.pythonPath.trim() !== "" } catch { return false } diff --git a/packages/opencode/src/altimate/tools/dbt-sql-graph.ts b/packages/opencode/src/altimate/tools/dbt-sql-graph.ts index f90581e7c8..6c04583fc1 100644 --- a/packages/opencode/src/altimate/tools/dbt-sql-graph.ts +++ b/packages/opencode/src/altimate/tools/dbt-sql-graph.ts @@ -132,7 +132,7 @@ const IDENT = String.raw`(?:"(?:[^"]|"")+"|\`(?:[^\`]|\`\`)+\`|\[[^\]]+\]|[A-Za- // A join's own relation, right after its JOIN keyword: `JOIN "db"."schema"."table" AS alias`. const JOINED_RELATION = new RegExp(String.raw`^((?:[A-Za-z]+\s+)*JOIN\s+)(${IDENT}(?:\s*\.\s*${IDENT})*)(?:\s+AS)?\s+(?!ON\b|USING\b)(${IDENT})(?=\s|$)`, "i") -/** SQL on one line: runs of whitespace become one space, except inside quoted strings and identifiers (`[…]` included). */ +/** SQL on one line: runs of whitespace become one space, except inside quoted strings and identifiers (`[…]` included), where only a line break does. */ function oneLine(sql: string): string { let out = "" for (let i = 0; i < sql.length; ) { @@ -142,7 +142,8 @@ function oneLine(sql: string): string { const close = ch === "[" ? "]" : ch let j = i + 1 while (j < sql.length && !(sql[j] === close && sql[j + 1] !== close)) j += sql[j] === close ? 2 : 1 - out += sql.slice(i, j + 1) + // Its spacing is kept, but not a line break: the name stays on one line. + out += sql.slice(i, j + 1).replace(/\r?\n/g, " ") i = j + 1 } else if (/\s/.test(ch)) { while (i < sql.length && /\s/.test(sql[i])) i++ diff --git a/packages/opencode/test/altimate/dbt-adapter-access.test.ts b/packages/opencode/test/altimate/dbt-adapter-access.test.ts index 3f353645b9..74ff3438ad 100644 --- a/packages/opencode/test/altimate/dbt-adapter-access.test.ts +++ b/packages/opencode/test/altimate/dbt-adapter-access.test.ts @@ -206,13 +206,23 @@ describe("dbtAdapterFor", () => { const saved = process.env.HOME try { await mkdir(path.join(home, ".altimate-code"), { recursive: true }) - await writeFile(path.join(home, ".altimate-code", "dbt.json"), "{}") + const config = path.join(home, ".altimate-code", "dbt.json") process.env.HOME = home dbtAdapterInternals.create = async () => ({ isInitialized: () => false, dispose: async () => {} }) + await writeFile(config, JSON.stringify({ projectRoot: project, pythonPath: "python3" })) const access = await dbtAdapterFor(ctx) expect(access.reason).toContain("Python that ~/.altimate-code/dbt.json names (python3)") expect(access.reason).toContain("Set pythonPath there") expect(access.reason).not.toContain("Activate the project's virtualenv") + // A config without a pythonPath, or one that doesn't parse, leaves the choice to discovery. + for (const text of ["{}", "{not json"]) { + await writeFile(config, text) + resetDbtAdapter() + useConfig() + const other = await dbtAdapterFor(ctx) + expect(other.reason).toContain("Activate the project's virtualenv") + expect(other.reason).not.toContain("Set pythonPath there") + } } finally { if (saved === undefined) delete process.env.HOME else process.env.HOME = saved diff --git a/packages/opencode/test/altimate/dbt-sql-graph.test.ts b/packages/opencode/test/altimate/dbt-sql-graph.test.ts index 234b2da71e..7dff10d186 100644 --- a/packages/opencode/test/altimate/dbt-sql-graph.test.ts +++ b/packages/opencode/test/altimate/dbt-sql-graph.test.ts @@ -875,6 +875,7 @@ describe("clauseName", () => { expect(clauseName({ type: "FILTER", expression: "WHERE code = 'a, b''s'\n AND \"x y\" = 1" })).toBe("WHERE code = 'a, b''s' AND \"x y\" = 1") expect(clauseName({ type: "LEFT_JOIN", expression: 'LEFT JOIN "db"."s"."Orders" AS "orders" ON 1 = 1' })).toBe("LEFT JOIN Orders AS orders ON 1 = 1") expect(clauseName({ type: "FILTER", expression: "WHERE [order status]]x] = 1" })).toBe("WHERE [order status]]x] = 1") + expect(clauseName({ type: "FILTER", expression: "WHERE msg <> 'a\nb'" })).toBe("WHERE msg <> 'a b'") // A name that needs its quotes keeps them. expect(clauseName({ type: "LEFT_JOIN", expression: 'LEFT JOIN "db"."s"."my table" AS "x y" ON 1 = 1' })).toBe('LEFT JOIN "my table" AS "x y" ON 1 = 1') }) From a14c731ed58dbaf069e19a7f6cce31fbc1b1a9f6 Mon Sep 17 00:00:00 2001 From: Harish Kumar Date: Fri, 20 Nov 2026 00:41:50 +0530 Subject: [PATCH 7/7] test: wait for the dbt adapter's creation to start before disposing it Creation now starts after the target folder is resolved, one more async step than the test's single macrotask allowed under load. Co-Authored-By: Claude Opus 5.5 --- .../opencode/test/altimate/dbt-lineage-adapter.test.ts | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/packages/opencode/test/altimate/dbt-lineage-adapter.test.ts b/packages/opencode/test/altimate/dbt-lineage-adapter.test.ts index 54c1b69d00..e616e5f982 100644 --- a/packages/opencode/test/altimate/dbt-lineage-adapter.test.ts +++ b/packages/opencode/test/altimate/dbt-lineage-adapter.test.ts @@ -435,12 +435,13 @@ describe("the dbt adapter sql_execute uses", () => { await disposeDbtAdapter() expect(disposed).toBe(1) - let finish!: (value: unknown) => void + let finish: ((value: unknown) => void) | undefined dbtAdapterInternals.create = () => new Promise((resolve) => (finish = resolve)) const starting = getDbtAdapter() - await new Promise((resolve) => setTimeout(resolve, 0)) + // Creation starts after the config and target-folder reads: wait until it is under way. + while (!finish) await new Promise((resolve) => setTimeout(resolve, 1)) await disposeDbtAdapter() - finish(adapter) + finish!(adapter) await starting await new Promise((resolve) => setTimeout(resolve, 0)) expect(disposed).toBe(2)