diff --git a/docs/docs/usage/tool.md b/docs/docs/usage/tool.md index d630c6ebd0..ebeb53b063 100644 --- a/docs/docs/usage/tool.md +++ b/docs/docs/usage/tool.md @@ -68,13 +68,15 @@ altimate-code tool daemon status altimate-code tool daemon stop ``` -`status` and `stop` locate the running daemon the same way `run` does: by project directory, CLI version, the `--connections` file (if any), whether workspace tools are on, and the caller's `ALTIMATE_CODE_CONN_*`, `ALTIMATE_DISABLE_SCHEMA_VALUES`, `ALTIMATE_INTEGRATIONS` and `ALTIMATE_DISABLE_WORKSPACE` environment variables. Run them with the same flags and variables as the call that started the daemon, or they will report no daemon running for this project even though one is. +`status` and `stop` locate the running daemon the same way `run` does: by project directory, CLI version, the `--connections` file (if any), whether workspace tools are on, and the caller's `ALTIMATE_CODE_CONN_*`, `ALTIMATE_DISABLE_SCHEMA_VALUES`, `ALTIMATE_INTEGRATIONS`, `ALTIMATE_DISABLE_WORKSPACE`, `VIRTUAL_ENV` and `CONDA_PREFIX` environment variables. The last two are part of the daemon's identity because, when `~/.altimate-code/dbt.json` sets no `pythonPath` and the project has no `.venv`, `venv` or `env` folder of its own, they pick the Python the daemon's dbt adapter runs dbt with: activating the project's virtualenv then gets a daemon that uses it rather than one started without it. A `pythonPath` in `dbt.json` always wins; change it there. Run them with the same flags and variables as the call that started the daemon, or they will report no daemon running for this project even though one is. | Setting | Description | |---|---| | `ALTIMATE_TOOL_DAEMON=0` | Always run in-process. | | `ALTIMATE_TOOL_DAEMON_IDLE_MS` | Idle time before the daemon exits (default `600000`). | +The tools that start the dbt adapter (`dbt_lineage`, `dbt_cte_profile`, `dbt_sql_graph`, `dbt_health`, `dbt_cost_estimate`, and `sql_execute` routed through dbt) have dbt write its artifacts to `/.altimate-code/` rather than the project's own `target/` in the `core` and `corecommand` integration modes: a dbt command rewrites `manifest.json` with compiled SQL for only what it compiled, so the project's `target/manifest.json` is left as your own `dbt compile` wrote it. The `fusion` and `cloud` modes read `/target` whatever dbt is told, so there dbt still writes to it. + The daemon listens on a local socket (a named pipe on Windows) under the Altimate Code state directory and removes it on exit. ## Claude Code routing diff --git a/packages/dbt-tools/package.json b/packages/dbt-tools/package.json index 66639efd2f..44174b14b5 100644 --- a/packages/dbt-tools/package.json +++ b/packages/dbt-tools/package.json @@ -9,7 +9,7 @@ "scripts": { "build": "bun build src/index.ts --outdir dist --target node --format esm && bun run script/copy-python.ts", "typecheck": "tsc --noEmit", - "test": "bun test test/cli.test.ts test/config.test.ts test/dbt-cli.test.ts test/dbt-resolve.test.ts test/adapter-bun.test.ts test/stale-manifest.test.ts test/build-integrity.test.ts --timeout 30000", + "test": "bun test test/cli.test.ts test/config.test.ts test/dbt-cli.test.ts test/dbt-resolve.test.ts test/adapter-bun.test.ts test/stale-manifest.test.ts test/build-integrity.test.ts test/adapter-target-path.test.ts --timeout 30000", "test:e2e": "bun test test/e2e/ --timeout 300000" }, "dependencies": { diff --git a/packages/dbt-tools/src/adapter.ts b/packages/dbt-tools/src/adapter.ts index 706d43ad6c..4786164c1a 100644 --- a/packages/dbt-tools/src/adapter.ts +++ b/packages/dbt-tools/src/adapter.ts @@ -81,8 +81,22 @@ function terminal(): DBTTerminal { } } -function env(cfg: Config): RuntimePythonEnvironment { +/** What `create` needs besides the saved config. */ +export type CreateOptions = { + /** + * The folder dbt writes its artifacts to (manifest.json, partial_parse.msgpack, run_results.json) for + * every command this adapter runs, given to dbt as DBT_TARGET_PATH. Unset, dbt writes to the project's + * own target-path, which is what a command run for the user should do. A caller that parses and + * compiles for its own reading passes a folder of its own: dbt rewrites manifest.json on every + * command, with compiled SQL only for what that command compiled, so writing to the project's target + * would take the compiled SQL of every other model away from whoever reads that manifest. + */ + targetPath?: string +} + +function env(cfg: Config, options: CreateOptions): RuntimePythonEnvironment { const vars = { ...process.env } + if (options.targetPath) vars.DBT_TARGET_PATH = options.targetPath return { pythonPath: cfg.pythonPath, getEnvironmentVariables: () => vars, @@ -98,7 +112,7 @@ export function hasModelLineage(adapter?: DBTProjectIntegrationAdapter): boolean return typeof target.getModelLineage === "function" } -export async function create(cfg: Config): Promise { +export async function create(cfg: Config, options: CreateOptions = {}): Promise { const config = configuration(cfg) const term = terminal() const factory = new DBTCommandFactory(config) @@ -106,7 +120,7 @@ export async function create(cfg: Config): Promise const http = new AltimateHttpClient(term, config) const client = new DbtIntegrationClient(http, term) - const runtime = env(cfg) + const runtime = env(cfg, options) const provider = { getCurrentEnvironment: () => runtime, onEnvironmentChanged: () => () => {}, diff --git a/packages/dbt-tools/test/adapter-target-path.test.ts b/packages/dbt-tools/test/adapter-target-path.test.ts new file mode 100644 index 0000000000..44fa984b73 --- /dev/null +++ b/packages/dbt-tools/test/adapter-target-path.test.ts @@ -0,0 +1,67 @@ +import { describe, test, expect } from "bun:test" +import { chmodSync, existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "fs" +import { tmpdir } from "os" +import { join } from "path" +import { create } from "../src/adapter" + +/** A stand-in `python` that records the DBT_TARGET_PATH each run sees and fails the bridge run. */ +function recordingPython(root: string) { + const log = join(root, "target-path.log") + const python = join(root, "python") + writeFileSync( + python, + [ + "#!/bin/sh", + `printf '%s\\n' "\${DBT_TARGET_PATH-unset}" >> '${log}'`, + 'case "$1" in', + ' --version) echo "Python 3.11.9"; exit 0 ;;', + ' -c) echo "1.10.0"; exit 0 ;;', + "esac", + "exit 3", + ].join("\n") + "\n", + ) + chmodSync(python, 0o755) + return { python, seen: () => (existsSync(log) ? readFileSync(log, "utf8").trim().split("\n") : []) } +} + +// The adapter's dbt commands rewrite manifest.json in the target folder they are given; a caller that +// compiles for its own reading gives one of its own, so the project's target/manifest.json is left alone. +describe.skipIf(process.platform === "win32")("create's target folder", () => { + const start = async (targetPath?: string) => { + const root = mkdtempSync(join(tmpdir(), "dbt-tools-target-")) + const project = join(root, "project") + mkdirSync(project) + writeFileSync(join(project, "dbt_project.yml"), "name: p\nversion: '1.0'\nprofile: p\n") + const { python, seen } = recordingPython(root) + const saved = process.env.DBT_TARGET_PATH + delete process.env.DBT_TARGET_PATH + try { + await create( + { projectRoot: project, pythonPath: python, dbtIntegration: "corecommand", queryLimit: 500 }, + targetPath ? { targetPath } : undefined, + ).then( + (adapter) => adapter.dispose(), + () => {}, + ) + // The host's own environment is not changed: other readers of DBT_TARGET_PATH still see the project's. + expect(process.env.DBT_TARGET_PATH).toBeUndefined() + return seen() + } finally { + if (saved === undefined) delete process.env.DBT_TARGET_PATH + else process.env.DBT_TARGET_PATH = saved + rmSync(root, { recursive: true, force: true }) + } + } + + test("every dbt process the adapter starts writes to the folder it is given", async () => { + const runs = await start("/tmp/project/target/.altimate-code") + expect(runs.length).toBeGreaterThan(0) + expect(new Set(runs)).toEqual(new Set(["/tmp/project/target/.altimate-code"])) + }) + + test("without one, dbt writes where the project says", async () => { + const runs = await start() + expect(runs.length).toBeGreaterThan(0) + expect(new Set(runs)).toEqual(new Set(["unset"])) + }) +}) diff --git a/packages/opencode/src/altimate/native/connections/register.ts b/packages/opencode/src/altimate/native/connections/register.ts index b404bd0444..39fcf6139f 100644 --- a/packages/opencode/src/altimate/native/connections/register.ts +++ b/packages/opencode/src/altimate/native/connections/register.ts @@ -6,6 +6,7 @@ * schema.inspect */ +import path from "path" import { register } from "../dispatcher" import { listSchemaTables } from "./schema-list" import * as Registry from "./registry" @@ -57,13 +58,15 @@ let dbtAdapter: any | null | undefined = undefined let dbtAdapterInflight: Promise | undefined /** The project root the cached attempt read from the dbt config; null when it found none. */ let dbtProjectRoot: string | null = null +/** The Python the cached attempt runs dbt with; null when it found no config. */ +let dbtPythonPath: string | null = null /** Test seams for the single-flight. Production leaves `readConfig` unset. */ export const dbtAdapterInternals: { /** Replaces the dbt config read, so a test can hold an attempt open. */ readConfig?: () => Promise /** Replaces dbt-tools' adapter creation, so a test can see which integration modes were tried. */ - create?: (config: any) => Promise + create?: (config: any, options?: { targetPath?: string }) => Promise /** Replaces the check for dbt-integration's lineage, which a test's fake adapter provides itself. */ hasModelLineage?: (adapter?: unknown) => boolean /** Adapter creation attempts since the last reset — what single-flight bounds. */ @@ -109,21 +112,27 @@ async function ensureDbtAdapter(): Promise { : await readDbtConfig() if (dbtAdapterInflight === mine) { dbtProjectRoot = typeof dbtConfig?.projectRoot === "string" ? dbtConfig.projectRoot : null + dbtPythonPath = typeof dbtConfig?.pythonPath === "string" ? dbtConfig.pythonPath : null } if (!dbtConfig) return publish(null) // Check if dbt_project.yml exists const fs = await import("fs") - const path = await import("path") if ( !fs.existsSync(path.join(dbtConfig.projectRoot, "dbt_project.yml")) ) { return publish(null) } - // Create the adapter + // Create the adapter. Its dbt commands write their artifacts to a folder of their own: the project's + // target/manifest.json is the user's (and other tools'), and a tool's compile or show would rewrite + // it with compiled SQL for only the nodes that command compiled. + // Only in the modes whose dbt-integration reads the artifacts back from where dbt wrote them. const create = dbtAdapterInternals.create ?? (await import("../../../../../dbt-tools/src/adapter")).create - return publish(await create(dbtConfig), dbtConfig.projectRoot) + const options = SEPARATE_TARGET_MODES.has(dbtConfig.dbtIntegration || "core") + ? { targetPath: await dbtToolTargetPath(dbtConfig.projectRoot) } + : {} + return publish(await create(dbtConfig, options), dbtConfig.projectRoot) } catch { // dbt-tools not available or config invalid — fall back to native return publish(null) @@ -135,6 +144,22 @@ async function ensureDbtAdapter(): Promise { return mine } +/** + * The dbt-integration modes (dbt.json's `dbtIntegration`, `core` when unset) that read the manifest back from the + * target path dbt itself reports, DBT_TARGET_PATH included. Fusion and dbt Cloud read `/target` whatever + * dbt was told, so they keep writing there: a separate folder would leave them reading a manifest nothing updates. + */ +export const SEPARATE_TARGET_MODES: ReadonlySet = new Set(["core", "corecommand"]) + +/** The folder, inside the project's target folder, where the tools' dbt adapter writes its dbt artifacts. */ +export const DBT_TOOL_TARGET_DIR = ".altimate-code" + +/** Where the tools' dbt adapter has dbt write manifest.json and the rest: never the project's own target folder. */ +export async function dbtToolTargetPath(projectRoot: string): Promise { + const { resolveDbtTargetPath } = await import("../../validators/validator-utils") + return path.join(await resolveDbtTargetPath(projectRoot), DBT_TOOL_TARGET_DIR) +} + /** Where a `warehouse`-less call would actually go. */ export type DefaultTarget = | { @@ -316,6 +341,11 @@ export async function getDbtProjectRoot(): Promise { } } +/** The Python the dbt adapter runs dbt with, once an attempt has read the dbt config. */ +export function getDbtPythonPath(): string | undefined { + return dbtPythonPath ?? undefined +} + /** Reset dbt adapter (for testing). */ export function resetDbtAdapter(): void { dbtAdapter = undefined @@ -326,6 +356,7 @@ export function resetDbtAdapter(): void { dbtAdapterInternals.attempts = 0 dbtAdapterInternals.writes = 0 dbtProjectRoot = null + dbtPythonPath = null } // --------------------------------------------------------------------------- diff --git a/packages/opencode/src/altimate/tool-cli/daemon.ts b/packages/opencode/src/altimate/tool-cli/daemon.ts index e28a10f510..86deb24422 100644 --- a/packages/opencode/src/altimate/tool-cli/daemon.ts +++ b/packages/opencode/src/altimate/tool-cli/daemon.ts @@ -44,8 +44,10 @@ export function resolveDirectory(directory: string): string { const CONN_ENV_PREFIX = "ALTIMATE_CODE_CONN_" // Variables a daemon reads once at spawn that change what it returns or exposes (value sampling, integration -// routing, workspace tools), so a caller with a different value must not be served by a daemon started without it. -const BEHAVIOUR_ENV = ["ALTIMATE_DISABLE_SCHEMA_VALUES", "ALTIMATE_INTEGRATIONS", "ALTIMATE_DISABLE_WORKSPACE"] +// routing, workspace tools, and the Python environment its dbt adapter runs dbt with), so a caller with a +// different value must not be served by a daemon started without it: a daemon started outside the project's +// virtualenv would otherwise go on answering that dbt didn't load after the caller activated it. +const BEHAVIOUR_ENV = ["ALTIMATE_DISABLE_SCHEMA_VALUES", "ALTIMATE_INTEGRATIONS", "ALTIMATE_DISABLE_WORKSPACE", "VIRTUAL_ENV", "CONDA_PREFIX"] /** A short digest of the caller's `ALTIMATE_CODE_CONN_*` variables and the behaviour variables above, empty when * none are set. A daemon reads them once at spawn, so callers with different values must not share one. */ diff --git a/packages/opencode/src/altimate/tool-zod-compat.ts b/packages/opencode/src/altimate/tool-zod-compat.ts index ccab23a3cd..15855e8380 100644 --- a/packages/opencode/src/altimate/tool-zod-compat.ts +++ b/packages/opencode/src/altimate/tool-zod-compat.ts @@ -176,6 +176,35 @@ export function zodToJsonSchema(schema: z.ZodType): JSONSchema7 { : rest) as JSONSchema7 } +// altimate_change start — argument errors that name the field +/** The value at `path` in `input`, undefined when any step is missing. */ +function valueAt(input: unknown, path: readonly PropertyKey[]): unknown { + let value = input + for (const key of path) { + if (typeof value !== "object" || value === null || !Object.hasOwn(value, key)) return undefined + value = (value as Record)[key] + } + return value +} + +/** + * Each issue of a failed zod parse as one clause that names its field: `change_type is required (expected + * one of "remove"|"rename")` for a missing one, `limit: Too small: expected number to be >=1` for a wrong one. + */ +export function describeZodIssues(error: z.ZodError, input: unknown): string { + const clauses = error.issues.map((issue) => { + if (issue.path.length === 0) return issue.message + const field = issue.path.map((p, i) => (typeof p === "number" ? `[${p}]` : `${i === 0 ? "" : "."}${String(p)}`)).join("") + if (valueAt(input, issue.path) === undefined) { + const expected = issue.message.replace(/^Invalid (?:input|option):\s*/, "").replace(/,\s*received undefined$/, "") + return `${field} is required (${expected})` + } + return `${field}: ${issue.message}` + }) + return clauses.length > 0 ? clauses.join("; ") : error.message +} +// altimate_change end + /** Build a `DefWithoutID` from a resolved legacy tool object (no Effect wrapper). */ function legacyDefToDef(legacy: LegacyToolDef): DefWithoutID { const validationSchema = zodToJsonSchema(legacy.parameters) @@ -200,24 +229,24 @@ function legacyDefToDef(legacy: LegacyToolDef): DefWithoutID { // tool needs hidden compatibility arguments that should not be advertised. jsonSchema: legacy.jsonSchema ?? validationSchema, // altimate_change end - ...(legacy.formatValidationError - ? { - // altimate_change start — the new Tool API decodes via Schema.declare, so the - // error reaching us is an Effect SchemaIssue (no zod `.issues`/`.path`), not a - // z.ZodError. Recover the original input from Effect SchemaError.issue.actual and - // re-run the zod schema to produce the real z.ZodError the legacy formatter - // expects; fall back to the original error if recovery is impossible. - formatValidationError: (error: unknown) => { - const actual = schemaErrorActual(error) - if (actual.found) { - const parsed = legacy.parameters.safeParse(actual.value) - if (!parsed.success) return legacy.formatValidationError!(parsed.error) - } - return legacy.formatValidationError!(error as z.ZodError) - }, - // altimate_change end - } - : {}), + // altimate_change start — the new Tool API decodes via Schema.declare, so the error reaching + // us is an Effect SchemaIssue (no zod `.issues`/`.path`), not a z.ZodError: on its own it reads + // "Expected " and names no field. Recover the original input from Effect + // SchemaError.issue.actual and re-run the zod schema to produce the real z.ZodError, for the + // legacy tool's own formatter or, when it has none, one that names each field and what it + // expects; fall back to the original error if recovery is impossible. + formatValidationError: (error: unknown) => { + const actual = schemaErrorActual(error) + if (actual.found) { + const parsed = legacy.parameters.safeParse(actual.value) + if (!parsed.success) + return legacy.formatValidationError + ? legacy.formatValidationError(parsed.error) + : describeZodIssues(parsed.error, actual.value) + } + return legacy.formatValidationError ? legacy.formatValidationError(error as z.ZodError) : String(error) + }, + // altimate_change end execute: (args: unknown, ctx: NewContext) => // altimate_change start — bridge the ambient Instance ALS into the Effect InstanceRef so // legacy tools whose execute calls async facades (Skill.get, Agent.list, …) — which resolve diff --git a/packages/opencode/src/altimate/tools/dbt-adapter-access.ts b/packages/opencode/src/altimate/tools/dbt-adapter-access.ts index 8c81cb0980..6beef71403 100644 --- a/packages/opencode/src/altimate/tools/dbt-adapter-access.ts +++ b/packages/opencode/src/altimate/tools/dbt-adapter-access.ts @@ -55,7 +55,7 @@ export async function dbtAdapterFor(ctx: Pick { + try { + const { path: configPath } = await import("../../../../dbt-tools/src/config") + const saved = JSON.parse(await fs.promises.readFile(configPath(), "utf8")) as { pythonPath?: unknown } | null + return typeof saved?.pythonPath === "string" && saved.pythonPath.trim() !== "" + } catch { + return false + } +} + /** `p` with its links resolved, or as given when it can't be. */ function real(p: string): string { try { diff --git a/packages/opencode/src/altimate/tools/dbt-cte-profile.ts b/packages/opencode/src/altimate/tools/dbt-cte-profile.ts index 512cb3851b..9b5a60af1d 100644 --- a/packages/opencode/src/altimate/tools/dbt-cte-profile.ts +++ b/packages/opencode/src/altimate/tools/dbt-cte-profile.ts @@ -1,3 +1,4 @@ +import crypto from "crypto" import fs from "fs" import path from "path" import z from "zod" @@ -7,6 +8,7 @@ import { projectRoot } from "../native/connections/registry" import { isQueryTimeoutError } from "@altimateai/drivers/types" import { executeQuery } from "../native/query-execution" import { dbtAdapterFor } from "./dbt-adapter-access" +import { untilAbort } from "./dbt-cost-estimate" import { buildCountQuery, detectCtes, extractRowCount, stripColumnList, unclosedJinjaBlock, unreadCteAfter, type CteInfo } from "./cte-detect" /** Folders a model search never enters. */ @@ -146,13 +148,111 @@ function nameMatches(list: CteInfo[], typed: string): CteInfo[] { return exact.length > 0 ? exact : list.filter((c) => bareName(c.name).toLowerCase() === typed.toLowerCase()) } +/** Test seams. */ +export const dbtCteProfileInternals = { + /** The trivial query whose time through dbt stands for dbt's own start-up, when dbt reports no timing. */ + baselineQuery: "select 1 as _profile_baseline", + /** How long a finished count waits for that measurement before it answers with its own time as is. */ + baselineWaitMs: 30_000, +} + +/** Per adapter: how long a trivial query takes through it, dbt's start-up with next to no warehouse work. */ +const baselines = new WeakMap>() + +/** A dbt timestamp in ms; dbt writes UTC, with or without the zone. */ +function utc(value: unknown): number { + if (typeof value !== "string" || !value) return NaN + return Date.parse(/(Z|[+-]\d\d:?\d\d)$/i.test(value) ? value : `${value}Z`) +} + +/** + * The warehouse time, in ms, of the execute step of the count query that carries `tag`, from the + * run_results.json dbt wrote for it; undefined when that file is not this query's. + */ +export function dbtQueryMs(targetDir: string, startedAt: number, tag: string): number | undefined { + let raw: any + try { + raw = JSON.parse(fs.readFileSync(path.join(targetDir, "run_results.json"), "utf-8")) + } catch { + return undefined + } + // Only the file this call's command wrote: a start after the call's, a `show` of one inline node, and + // this call's tag in its compiled SQL. Other commands (another call's count of the same CTE, another + // daemon on the project) write the same file, and only the tag tells two runs of one query apart. + const invoked = utc(raw?.metadata?.invocation_started_at ?? raw?.metadata?.generated_at) + if (!Number.isFinite(invoked) || invoked < startedAt) return undefined + if (raw?.args?.which !== undefined && raw.args.which !== "show") return undefined + const results = Array.isArray(raw?.results) ? raw.results : [] + if (results.length !== 1) return undefined + const node = results[0] + if (typeof node?.compiled_code !== "string" || !node.compiled_code.includes(tag)) return undefined + const execute = Array.isArray(node?.timing) ? node.timing.find((t: any) => t?.name === "execute") : undefined + const span = utc(execute?.completed_at) - utc(execute?.started_at) + // Only the execute step: the node's execution_time also counts compiling it. + return Number.isFinite(span) && span >= 0 ? Math.round(span) : undefined +} + +/** The comment a count query starts with, unique to the call, by which its run_results.json is known. */ +export function profileTag(): string { + return `altimate-code cte profile ${crypto.randomUUID()}` +} + +/** `work`'s value, or undefined once `ms` have passed first; `work` goes on either way. */ +function within(work: Promise, ms: number): Promise { + let timer: ReturnType | undefined + const late = new Promise((resolve) => (timer = setTimeout(() => resolve(undefined), ms))) + return Promise.race([work, late]).finally(() => clearTimeout(timer)) +} + +type QueryTime = { source: "dbt" | "baseline" | "wall"; queryMs: number; startupMs?: number } + +/** + * The count query's time on the warehouse, without dbt's start-up, which a command-line dbt pays on every + * query (seconds, against milliseconds for a small CTE). First dbt's own timing of the query, from the + * run_results.json it wrote; else the call's time less a trivial query's time through the same adapter, + * measured once per adapter; else the call's time as is. + */ +async function queryTime(adapter: any, call: { startedAt: number; totalMs: number; tag: string }, ctx: Pick): Promise { + const dir = typeof adapter.getTargetPath === "function" ? adapter.getTargetPath() : undefined + const fromDbt = typeof dir === "string" && dir ? dbtQueryMs(dir, call.startedAt, call.tag) : undefined + if (fromDbt !== undefined) return { source: "dbt", queryMs: Math.min(fromDbt, call.totalMs) } + let baseline = baselines.get(adapter) + if (!baseline) { + // Shared by the calls that wait for it, so no one call's abort cancels it: each call stops waiting at its + // own abort, and the measurement is bounded by the query timeout. + const measured: Promise = (async () => { + const started = performance.now() + try { + await executeQuery({ type: "dbt" }, {}, { sessionID: ctx.sessionID }, () => + adapter.immediatelyExecuteSQLWithLimit(dbtCteProfileInternals.baselineQuery, "", 1), + ) + return Math.round(performance.now() - started) + } catch { + return undefined + } + })() + // A failed measurement is not kept: the next call measures again. + void measured.then((ms) => { + if (ms === undefined && baselines.get(adapter) === measured) baselines.delete(adapter) + }) + baseline = measured + baselines.set(adapter, baseline) + } + // A measurement that stalls doesn't hold a count that has finished: past the wait, the count's own time is given. + const pending = baseline + const startupMs = await untilAbort(() => within(pending, dbtCteProfileInternals.baselineWaitMs), ctx.abort) + if (startupMs === undefined) return { source: "wall", queryMs: call.totalMs } + const startup = Math.min(startupMs, call.totalMs) + return { source: "baseline", queryMs: call.totalMs - startup, startupMs: startup } +} + function failure(title: string, output: string, extra: Record = {}) { return { title: `dbt_cte_profile: ${title}`, metadata: { success: false, error: output, ...extra }, output } } export const DbtCteProfileTool = Tool.define("dbt_cte_profile", { description: - "List a dbt model's CTEs, or time one CTE with a row-count query. Without `cte` or `position` it returns the CTE names in source order (name, line, position) and runs nothing, so a read-only call works. With one of them it runs `SELECT COUNT(*)` over that CTE and the CTEs before it in the same WITH clause on the warehouse, and returns the elapsed milliseconds and the row count; that run needs write permission (`--allow-write`) because it starts the dbt adapter and compiles the model. Each run recomputes the earlier CTEs, so a model of N CTEs costs N queries. Only the model's own top-level WITH clause is read: a WITH inside a CTE body is part of that CTE. The list stops at a CTE whose name Jinja builds, and `incomplete` is set in the metadata then; a CTE whose count query reads is_incremental() or this, in its own SQL, an earlier CTE or the model header, is not run, because the count runs outside the model. Macros the query calls are not opened: one that reads is_incremental() inside is not caught, so treat the count of such a model as a full-history count.", + "List a dbt model's CTEs, or time one CTE with a row-count query. Without `cte` or `position` it returns the CTE names in source order (name, line, position) and runs nothing, so a read-only call works. With one of them it runs `SELECT COUNT(*)` over that CTE and the CTEs before it in the same WITH clause on the warehouse, and returns the row count and the query's milliseconds on the warehouse (`queryMs`): dbt's own timing of the query's execute step where dbt reports one (run_results.json), else the call's time less that of a trivial query through dbt, so a command-line dbt's start-up of a second or more per query is not counted; `totalMs` is the count query's whole call through dbt, start-up included, and `timing` says which measure `queryMs` is (dbt, baseline, or wall when neither was available); that run needs write permission (`--allow-write`) because it starts the dbt adapter and compiles the model. Each run recomputes the earlier CTEs, so a model of N CTEs costs N queries. Only the model's own top-level WITH clause is read: a WITH inside a CTE body is part of that CTE. The list stops at a CTE whose name Jinja builds, and `incomplete` is set in the metadata then; a CTE whose count query reads is_incremental() or this, in its own SQL, an earlier CTE or the model header, is not run, because the count runs outside the model. Macros the query calls are not opened: one that reads is_incremental() inside is not caught, so treat the count of such a model as a full-history count.", parameters: z.object({ model: z .string() @@ -249,18 +349,40 @@ export const DbtCteProfileTool = Tool.define("dbt_cte_profile", { if (!access.adapter) return failure("not run", access.reason ?? "the dbt adapter is not available", { model: args.model, cte: name, refusal: access.refusal }) const adapter = access.adapter ctx.abort.throwIfAborted() + const startedAt = Date.now() const started = performance.now() try { // The adapter has no cancel of its own: the run's timeout, budget and abort end the wait, not the query. + // The tag makes this run's compiled SQL its own, so its run_results.json can't be taken for another's. + const tag = profileTag() const result: { data?: Record[] } = await executeQuery({ type: "dbt" }, {}, { signal: ctx.abort, sessionID: ctx.sessionID }, () => - adapter.immediatelyExecuteSQLWithLimit(query, path.basename(modelPath, ".sql"), 1), + adapter.immediatelyExecuteSQLWithLimit(`/* ${tag} */\n${query}`, path.basename(modelPath, ".sql"), 1), ) - const queryMs = Math.round(performance.now() - started) + const totalMs = Math.round(performance.now() - started) const rows = extractRowCount(result?.data) + const timing = await queryTime(adapter, { startedAt, totalMs, tag }, ctx) + const rowText = `${rows} row${rows === 1 ? "" : "s"}` + const output = + timing.source === "dbt" + ? `${name}: ${timing.queryMs} ms on the warehouse (dbt's own timing of the query), ${rowText}; ${totalMs} ms in all with dbt's start-up` + : timing.source === "baseline" + ? `${name}: ${timing.queryMs} ms on the warehouse (${totalMs} ms in all, less ${timing.startupMs} ms that a trivial query takes through dbt), ${rowText}` + : `${name}: ${totalMs} ms, ${rowText} (dbt's start-up included: no query timing of its own was available)` return { - title: `dbt_cte_profile: ${name} ${queryMs} ms`, - metadata: { success: true, model: args.model, cte: name, index: target.index, position: ctes.indexOf(target), queryMs, rows }, - output: `${name}: ${queryMs} ms, ${rows} row${rows === 1 ? "" : "s"}`, + title: `dbt_cte_profile: ${name} ${timing.queryMs} ms`, + metadata: { + success: true, + model: args.model, + cte: name, + index: target.index, + position: ctes.indexOf(target), + queryMs: timing.queryMs, + totalMs, + timing: timing.source, + ...(timing.startupMs !== undefined && { startupMs: timing.startupMs }), + rows, + }, + output, } } catch (e) { if (ctx.abort.aborted) throw e diff --git a/packages/opencode/src/altimate/tools/dbt-health.ts b/packages/opencode/src/altimate/tools/dbt-health.ts index b91c5d883d..6ca9f8541c 100644 --- a/packages/opencode/src/altimate/tools/dbt-health.ts +++ b/packages/opencode/src/altimate/tools/dbt-health.ts @@ -267,7 +267,7 @@ function failure(title: string, output: string, extra: Record = export const DbtHealthTool = Tool.define("dbt_health", { description: - "DataPilot project health insights for the models of the dbt project (hard-coded references, missing tests or documentation, naming and structure issues, and so on). Uses the project's dbt integration (starts it, so it needs write permission) for a fresh manifest, then runs DataPilot (the `altimate-datapilot-cli` Python package, which must be installed in the project's Python) over the manifest, and the catalog when target/catalog.json exists. Returns `insights`: uniqueId, path, name, severity, message and recommendation for each, plus the model's absolute `file` and `properties` file for the project's own models. When dbt can't parse the project the answer is a failure with the parse errors (no insights from the last manifest); a failed project check alone is answered from the last good manifest, labelled `stale_manifest`. A very large result is cut at a byte limit: `truncated` is then true, and `total` counts every insight.", + "DataPilot project health insights for the models of the dbt project (hard-coded references, missing tests or documentation, naming and structure issues, and so on). Uses the project's dbt integration (starts it, so it needs write permission) for a fresh manifest, then runs DataPilot (the `altimate-datapilot-cli` Python package, which must be installed in the project's Python) over the manifest, and the catalog when the project's target/catalog.json exists. Returns `insights`: uniqueId, path, name, severity, message and recommendation for each, plus the model's absolute `file` and `properties` file for the project's own models. When dbt can't parse the project the answer is a failure with the parse errors (no insights from the last manifest); a failed project check alone is answered from the last good manifest, labelled `stale_manifest`. A very large result is cut at a byte limit: `truncated` is then true, and `total` counts every insight.", parameters: z.object({ config_path: z .string() @@ -346,8 +346,13 @@ export const DbtHealthTool = Tool.define("dbt_health", { age_seconds: Math.max(0, Math.round((Date.now() - written) / 1000)), } } - const catalogFile = path.join(path.dirname(manifest), "catalog.json") - const catalog = fs.existsSync(catalogFile) ? catalogFile : "" + // The adapter's dbt writes to a target folder of its own, where `dbt docs generate` never puts a catalog: + // the one beside its manifest, else the project's own. + const { resolveDbtTargetPath } = await import("../validators/validator-utils") + const catalog = + [path.join(path.dirname(manifest), "catalog.json"), path.join(await resolveDbtTargetPath(root), "catalog.json")].find((file) => + fs.existsSync(file), + ) ?? "" let configPath = "" if (args.config_path) { configPath = path.resolve(root, args.config_path) diff --git a/packages/opencode/src/altimate/tools/dbt-sql-graph.ts b/packages/opencode/src/altimate/tools/dbt-sql-graph.ts index c3a950902c..6c04583fc1 100644 --- a/packages/opencode/src/altimate/tools/dbt-sql-graph.ts +++ b/packages/opencode/src/altimate/tools/dbt-sql-graph.ts @@ -117,6 +117,68 @@ export function mergeColumns( return Object.fromEntries(merged) } +/** Longest name a join, filter, group or sort node is given; its full expression stays in `sql`. */ +export const CLAUSE_NAME_CHARS = 80 + +/** An identifier as written, without its quotes. */ +function unquote(part: string): string { + const q = part[0] + if ((q === '"' || q === "`") && part.endsWith(q) && part.length > 1) return part.slice(1, -1).replaceAll(q + q, q) + if (q === "[" && part.endsWith("]")) return part.slice(1, -1) + return part +} + +const IDENT = String.raw`(?:"(?:[^"]|"")+"|\`(?:[^\`]|\`\`)+\`|\[[^\]]+\]|[A-Za-z_][\w$]*)` +// A join's own relation, right after its JOIN keyword: `JOIN "db"."schema"."table" AS alias`. +const JOINED_RELATION = new RegExp(String.raw`^((?:[A-Za-z]+\s+)*JOIN\s+)(${IDENT}(?:\s*\.\s*${IDENT})*)(?:\s+AS)?\s+(?!ON\b|USING\b)(${IDENT})(?=\s|$)`, "i") + +/** SQL on one line: runs of whitespace become one space, except inside quoted strings and identifiers (`[…]` included), where only a line break does. */ +function oneLine(sql: string): string { + let out = "" + for (let i = 0; i < sql.length; ) { + const ch = sql[i] + if (ch === "'" || ch === '"' || ch === "`" || ch === "[") { + // To the closing quote (`]` for a bracketed name); a doubled one is an escaped one. + const close = ch === "[" ? "]" : ch + let j = i + 1 + while (j < sql.length && !(sql[j] === close && sql[j + 1] !== close)) j += sql[j] === close ? 2 : 1 + // Its spacing is kept, but not a line break: the name stays on one line. + out += sql.slice(i, j + 1).replace(/\r?\n/g, " ") + i = j + 1 + } else if (/\s/.test(ch)) { + while (i < sql.length && /\s/.test(sql[i])) i++ + out += " " + } else { + out += ch + i++ + } + } + return out.trim() +} + +/** + * A name for a node the backend leaves unnamed (a join, filter, group or sort): its clause on one line, a relation + * written as its own name (`LEFT JOIN payments ON …`, `WHERE status <> 'returned'`), cut at CLAUSE_NAME_CHARS. + * Undefined when the node has no clause to show. + */ +export function clauseName(details: Pick): string | undefined { + let text = oneLine(details.expression ?? details.sql ?? "") + if (!text) return undefined + // A joined relation keeps its last name part; its alias goes when it only repeats that name. Nothing else in + // the clause is rewritten: an `AS` in a condition (a CAST) is not a relation. + text = text.replace(JOINED_RELATION, (_all, join: string, relation: string, alias: string) => { + // A name that needs its quotes (a space, a keyword-like spelling) keeps them; a plain one is shown bare. + const shown = (part: string) => (/^[A-Za-z_][\w$]*$/.test(unquote(part)) ? unquote(part) : part) + const last = relation.match(new RegExp(IDENT, "g"))!.at(-1)! + // Exactly the same spelling: a quoted alias that differs in case can name another relation. + return `${join}${unquote(last) === unquote(alias) ? shown(last) : `${shown(last)} AS ${shown(alias)}`}` + }) + // The backend writes an inner join as a bare JOIN; its type says which join it is. + const kind = /^([A-Z]+)_JOIN$/i.exec(details.type ?? "")?.[1] + if (kind && /^JOIN\b/i.test(text)) text = `${kind.toUpperCase()} ${text}` + return text.length > CLAUSE_NAME_CHARS ? `${text.slice(0, CLAUSE_NAME_CHARS - 1).trimEnd()}…` : text +} + /** * The backend names the model's own select `__final_select__`: name it after the model (or ` (final select)` when the SQL has a node of that name), tie each * relation node to the dbt node it stands for, and drop the layout positions the list doesn't use. @@ -177,9 +239,10 @@ export function remapSqlLineage( const nodes: SqlGraphNode[] = Object.entries(details).map(([key, d]) => { const merged = { ...d, ...mapping[key] } return { - // The key is the node's identity (edges use it); the name is only what is shown. + // The key is the node's identity (edges use it); the name is only what is shown. A join, filter, group + // or sort has no name from the backend, only a generated key (`Where_1405…`): it is named by its clause. id: key, - name: key === finalKey ? finalKey : (merged.name ?? key), + name: key === finalKey ? finalKey : (merged.name ?? clauseName(merged) ?? key), type: merged.type, ...(merged.nodeId ? { nodeId: merged.nodeId } : {}), sql: merged.sql ?? merged.expression ?? "", diff --git a/packages/opencode/test/altimate/dbt-adapter-access.test.ts b/packages/opencode/test/altimate/dbt-adapter-access.test.ts index 3f47caa7f6..74ff3438ad 100644 --- a/packages/opencode/test/altimate/dbt-adapter-access.test.ts +++ b/packages/opencode/test/altimate/dbt-adapter-access.test.ts @@ -2,14 +2,14 @@ // one gate the new dbt-adapter-backed tools go through, and the API helper for new endpoint files. The // dbt adapter is replaced through `dbtAdapterInternals`, so these tests need no Python or dbt. import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, test } from "bun:test" -import { chmod, mkdtemp, rm, writeFile } from "node:fs/promises" +import { chmod, mkdir, mkdtemp, rm, writeFile } from "node:fs/promises" import { existsSync, readFileSync, realpathSync } from "node:fs" import os from "node:os" import path from "node:path" import { Effect } from "effect" import type { Tool } from "@/tool/tool" import { SessionID, MessageID } from "@/session/schema" -import { dbtAdapterInternals, resetDbtAdapter } from "@/altimate/native/connections/register" +import { DBT_TOOL_TARGET_DIR, dbtAdapterInternals, dbtToolTargetPath, resetDbtAdapter } from "@/altimate/native/connections/register" import { dbtAdapterAccessInternals, dbtAdapterFor } from "@/altimate/tools/dbt-adapter-access" import { resultFailed } from "@/altimate/mcp-server/adapter" import { EXPOSED, MCP_OPEN_WRITE_TOOLS, WRITE_TOOLS } from "@/altimate/tool-cli/allow-list" @@ -135,7 +135,11 @@ describe("feature tools in the allow-list", () => { describe("dbtAdapterFor", () => { let created: number + // The default target folder is the project's own `target`: an inherited DBT_TARGET_PATH would move it. + let inheritedTargetPath: string | undefined beforeEach(() => { + inheritedTargetPath = process.env.DBT_TARGET_PATH + delete process.env.DBT_TARGET_PATH created = 0 useConfig() dbtAdapterInternals.create = async () => { @@ -143,6 +147,10 @@ describe("dbtAdapterFor", () => { return { isInitialized: () => true, dispose: async () => {} } } }) + afterEach(() => { + if (inheritedTargetPath === undefined) delete process.env.DBT_TARGET_PATH + else process.env.DBT_TARGET_PATH = inheritedTargetPath + }) test("refuses a read-only call and never creates the adapter", async () => { const access = await dbtAdapterFor({ ...ctx, extra: { forceReadOnly: true } }) @@ -180,7 +188,88 @@ describe("dbtAdapterFor", () => { test("says when the adapter didn't load the project", async () => { dbtAdapterInternals.create = async () => ({ isInitialized: () => false, dispose: async () => {} }) const access = await dbtAdapterFor(ctx) - expect(access).toEqual({ refusal: "not_loaded", reason: "the dbt adapter didn't load the project" }) + expect(access.refusal).toBe("not_loaded") + expect(access.reason).toStartWith("the dbt adapter didn't load the project: ") + }) + + test("a refusal for an adapter that didn't load names the Python dbt ran with and how to fix it", async () => { + dbtAdapterInternals.create = async () => ({ isInitialized: () => false, dispose: async () => {} }) + const access = await dbtAdapterFor(ctx) + // The Python the config chose: dbt missing from it is the usual cause. + expect(access.reason).toContain("(python3)") + expect(access.reason).toContain("Activate the project's virtualenv") + expect(access.reason).toContain("altimate-code tool daemon stop") + }) + + test("with a saved dbt config, the refusal says to change its pythonPath, which wins over any virtualenv", async () => { + const home = await mkdtemp(path.join(os.tmpdir(), "dbt-adapter-access-home-")) + const saved = process.env.HOME + try { + await mkdir(path.join(home, ".altimate-code"), { recursive: true }) + const config = path.join(home, ".altimate-code", "dbt.json") + process.env.HOME = home + dbtAdapterInternals.create = async () => ({ isInitialized: () => false, dispose: async () => {} }) + await writeFile(config, JSON.stringify({ projectRoot: project, pythonPath: "python3" })) + const access = await dbtAdapterFor(ctx) + expect(access.reason).toContain("Python that ~/.altimate-code/dbt.json names (python3)") + expect(access.reason).toContain("Set pythonPath there") + expect(access.reason).not.toContain("Activate the project's virtualenv") + // A config without a pythonPath, or one that doesn't parse, leaves the choice to discovery. + for (const text of ["{}", "{not json"]) { + await writeFile(config, text) + resetDbtAdapter() + useConfig() + const other = await dbtAdapterFor(ctx) + expect(other.reason).toContain("Activate the project's virtualenv") + expect(other.reason).not.toContain("Set pythonPath there") + } + } finally { + if (saved === undefined) delete process.env.HOME + else process.env.HOME = saved + await rm(home, { recursive: true, force: true }) + } + }) + + test("the tools' adapter is created with a dbt target folder of its own, inside the project's target", async () => { + let options: { targetPath?: string } | undefined + dbtAdapterInternals.create = async (_config, given) => { + options = given + return { isInitialized: () => true, dispose: async () => {} } + } + await dbtAdapterFor(ctx) + expect(options).toEqual({ targetPath: path.join(project, "target", DBT_TOOL_TARGET_DIR) }) + }) + + test("Fusion and dbt Cloud, which read /target whatever dbt is told, keep dbt writing there", async () => { + const seen: ({ targetPath?: string } | undefined)[] = [] + dbtAdapterInternals.create = async (_config, given) => { + seen.push(given) + return { isInitialized: () => true, dispose: async () => {} } + } + for (const dbtIntegration of ["fusion", "cloud", "core", ""]) { + resetDbtAdapter() + dbtAdapterInternals.readConfig = async () => ({ projectRoot: project, pythonPath: "python3", dbtIntegration, queryLimit: 500 }) + await dbtAdapterFor(ctx) + } + const own = { targetPath: path.join(project, "target", DBT_TOOL_TARGET_DIR) } + // An unset mode is dbt-integration's default, core. + expect(seen).toEqual([{}, {}, own, own]) + }) + + test("the tools' target folder follows the project's own target-path and DBT_TARGET_PATH", async () => { + const yml = path.join(project, "dbt_project.yml") + const saved = readFileSync(yml, "utf8") + const env = process.env.DBT_TARGET_PATH + try { + await writeFile(yml, `${saved}target-path: "build"\n`) + expect(await dbtToolTargetPath(project)).toBe(path.join(project, "build", DBT_TOOL_TARGET_DIR)) + process.env.DBT_TARGET_PATH = path.join(project, "elsewhere") + expect(await dbtToolTargetPath(project)).toBe(path.join(project, "elsewhere", DBT_TOOL_TARGET_DIR)) + } finally { + await writeFile(yml, saved) + if (env === undefined) delete process.env.DBT_TARGET_PATH + else process.env.DBT_TARGET_PATH = env + } }) test("says there is no dbt project when the config finds none", async () => { @@ -207,7 +296,9 @@ describe("dbtAdapterFor", () => { throw new Error("dbt-integration missing") } const access = await dbtAdapterFor(ctx) - expect(access).toEqual({ refusal: "not_started", reason: "the dbt adapter didn't start" }) + expect(access.refusal).toBe("not_started") + expect(access.reason).toStartWith("the dbt adapter didn't start: ") + expect(access.reason).toContain("(python3)") expect(created).toBe(1) }) diff --git a/packages/opencode/test/altimate/dbt-cte-profile.test.ts b/packages/opencode/test/altimate/dbt-cte-profile.test.ts index 002b1193fc..6e747dbc51 100644 --- a/packages/opencode/test/altimate/dbt-cte-profile.test.ts +++ b/packages/opencode/test/altimate/dbt-cte-profile.test.ts @@ -8,7 +8,7 @@ import { Effect } from "effect" import type { Tool } from "@/tool/tool" import { SessionID, MessageID } from "@/session/schema" import { dbtAdapterInternals, resetDbtAdapter } from "@/altimate/native/connections/register" -import { DbtCteProfileTool } from "@/altimate/tools/dbt-cte-profile" +import { DbtCteProfileTool, dbtCteProfileInternals, dbtQueryMs } from "@/altimate/tools/dbt-cte-profile" import { initTool } from "./tool-fixture" const ctx: Tool.Context = { @@ -41,7 +41,11 @@ select * from final let project: string let queries: { sql: string; name: string; limit: number }[] +/** The trivial queries the tool times dbt's start-up with, kept apart from the count queries. */ +let baselines: { sql: string; name: string; limit: number }[] let respond: (sql: string) => Promise +/** The adapter's dbt target folder, where a real dbt writes run_results.json; undefined: the adapter has none. */ +let adapterTarget: string | undefined function useAdapter() { dbtAdapterInternals.readConfig = async () => ({ @@ -53,8 +57,9 @@ function useAdapter() { dbtAdapterInternals.create = async () => ({ isInitialized: () => true, dispose: async () => {}, + getTargetPath: () => adapterTarget, immediatelyExecuteSQLWithLimit: async (sql: string, name: string, limit: number) => { - queries.push({ sql, name, limit }) + ;(sql === dbtCteProfileInternals.baselineQuery ? baselines : queries).push({ sql, name, limit }) return respond(sql) }, }) @@ -104,7 +109,9 @@ afterAll(async () => { }) beforeEach(() => { + adapterTarget = undefined queries = [] + baselines = [] respond = async () => ({ columnNames: ["_PROFILE_COUNT"], data: [{ _PROFILE_COUNT: 42 }] }) resetDbtAdapter() useAdapter() @@ -505,3 +512,195 @@ describe("dbt_cte_profile run", () => { expect(queries).toEqual([]) }) }) + +/** What dbt writes for one `dbt show --inline`: its start, and the execute step's span. */ +function runResults(opts: { started: Date; executeMs: number; compiled: string; which?: string }) { + const iso = (d: Date) => d.toISOString().replace("Z", "123Z") + const exec0 = new Date(opts.started.getTime() + 1500) + return JSON.stringify({ + metadata: { invocation_started_at: iso(opts.started), generated_at: iso(new Date(exec0.getTime() + opts.executeMs + 5)) }, + args: { which: opts.which ?? "show" }, + results: [ + { + unique_id: "sql_operation.jaffle_shop.inline_query", + execution_time: (opts.executeMs + 2) / 1000, + compiled_code: opts.compiled, + timing: [ + { name: "compile", started_at: iso(new Date(exec0.getTime() - 2)), completed_at: iso(exec0) }, + { name: "execute", started_at: iso(exec0), completed_at: iso(new Date(exec0.getTime() + opts.executeMs)) }, + ], + }, + ], + }) +} + +describe("dbt_cte_profile timing", () => { + let target: string + beforeEach(async () => { + target = await mkdtemp(path.join(os.tmpdir(), "dbt-cte-profile-target-")) + adapterTarget = target + }) + afterEach(async () => { + await rm(target, { recursive: true, force: true }) + }) + + const compiledOf = (sql: string) => sql.replace(/\{\{[^}]*\}\}/g, "x").trim() + const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms)) + + test("the time is dbt's own timing of the query, without dbt's start-up", async () => { + respond = async (sql) => { + const started = new Date() + // A command-line dbt spends most of the call starting up. + await sleep(60) + await writeFile(path.join(target, "run_results.json"), runResults({ started, executeMs: 7, compiled: compiledOf(sql) })) + return { columnNames: ["_PROFILE_COUNT"], data: [{ _PROFILE_COUNT: 42 }] } + } + const tool = await initTool(DbtCteProfileTool) + const result = await tool.execute({ model: "customers", cte: "orders" }, ctx) + expect(result.metadata).toMatchObject({ success: true, timing: "dbt", queryMs: 7, rows: 42 }) + expect(result.metadata.totalMs).toBeGreaterThanOrEqual(60) + expect(result.output).toContain("orders: 7 ms on the warehouse") + expect(result.output).toContain(`${result.metadata.totalMs} ms in all`) + expect(queries).toHaveLength(1) + expect(queries[0].sql).toMatch(/^\/\* altimate-code cte profile [0-9a-f-]{36} \*\/\n/) + expect(baselines).toEqual([]) + }) + + test("without dbt's timing, a trivial query's time through dbt is taken off, measured once per adapter", async () => { + respond = async (sql) => { + await sleep(sql === dbtCteProfileInternals.baselineQuery ? 40 : 100) + return { columnNames: ["_PROFILE_COUNT"], data: [{ _PROFILE_COUNT: 42 }] } + } + const tool = await initTool(DbtCteProfileTool) + const first = await tool.execute({ model: "customers", cte: "orders" }, ctx) + const second = await tool.execute({ model: "customers", cte: "final" }, ctx) + for (const result of [first, second]) { + expect(result.metadata.timing).toBe("baseline") + expect(result.metadata.queryMs).toBe(result.metadata.totalMs - result.metadata.startupMs) + expect(result.metadata.startupMs).toBeGreaterThanOrEqual(35) + expect(result.metadata.queryMs).toBeLessThan(result.metadata.totalMs) + } + expect(baselines).toEqual([{ sql: dbtCteProfileInternals.baselineQuery, name: "", limit: 1 }]) + }) + + test("a run_results.json from before the call, of another command, or without the call's tag is not this query's timing", async () => { + const tag = "altimate-code cte profile t-1" + const mine = `/* ${tag} */ select 1` + const before = new Date(Date.now() - 60_000) + await writeFile(path.join(target, "run_results.json"), runResults({ started: before, executeMs: 3, compiled: mine })) + expect(dbtQueryMs(target, Date.now(), tag)).toBeUndefined() + const now = Date.now() + await writeFile(path.join(target, "run_results.json"), runResults({ started: new Date(now + 5), executeMs: 3, compiled: "/* altimate-code cte profile t-2 */ select 1" })) + expect(dbtQueryMs(target, now, tag)).toBeUndefined() + await writeFile(path.join(target, "run_results.json"), runResults({ started: new Date(now + 5), executeMs: 3, compiled: mine, which: "build" })) + expect(dbtQueryMs(target, now, tag)).toBeUndefined() + await writeFile(path.join(target, "run_results.json"), runResults({ started: new Date(now + 5), executeMs: 3, compiled: mine })) + expect(dbtQueryMs(target, now, tag)).toBe(3) + // Without the execute step's timing there is no warehouse time: execution_time also counts the compile. + const noTiming = JSON.parse(runResults({ started: new Date(now + 5), executeMs: 3, compiled: mine })) + noTiming.results[0].timing = [] + await writeFile(path.join(target, "run_results.json"), JSON.stringify(noTiming)) + expect(dbtQueryMs(target, now, tag)).toBeUndefined() + }) + + test("dbt's timestamps without a zone are read as UTC", async () => { + const now = Date.now() + const naive = runResults({ started: new Date(now + 5), executeMs: 9, compiled: "/* t */ select 1" }).replaceAll(/(\d{3})123Z/g, "$1123") + await writeFile(path.join(target, "run_results.json"), naive) + expect(dbtQueryMs(target, now, "t")).toBe(9) + }) + + test("two calls counting the same CTE at once each take only their own run's timing", async () => { + // A's dbt writes its run_results.json, then B's overwrites it before A's call returns. + let releaseA = () => {} + const aWritten = new Promise((resolve) => (releaseA = resolve)) + let bDone = () => {} + const bWritten = new Promise((resolve) => (bDone = resolve)) + let calls = 0 + respond = async (sql) => { + if (sql === dbtCteProfileInternals.baselineQuery) return { columnNames: ["N"], data: [{ N: 1 }] } + const which = calls++ === 0 ? "A" : "B" + const started = new Date() + await sleep(5) + if (which === "A") { + await writeFile(path.join(target, "run_results.json"), runResults({ started, executeMs: 500, compiled: compiledOf(sql) })) + releaseA() + await bWritten + } else { + await aWritten + await writeFile(path.join(target, "run_results.json"), runResults({ started, executeMs: 3, compiled: compiledOf(sql) })) + bDone() + } + return { columnNames: ["_PROFILE_COUNT"], data: [{ _PROFILE_COUNT: 42 }] } + } + const tool = await initTool(DbtCteProfileTool) + const [a, b] = await Promise.all([ + tool.execute({ model: "customers", cte: "orders" }, ctx), + tool.execute({ model: "customers", cte: "orders" }, ctx), + ]) + // Whichever call ran first wrote 500 ms and saw it overwritten: it falls back to the start-up baseline + // rather than report the other run's 3 ms, and only the run that wrote 3 ms reports it. + const timings = [a.metadata, b.metadata].map((m) => `${m.timing}:${m.timing === "dbt" ? m.queryMs : "-"}`).sort() + expect(timings).toEqual(["baseline:-", "dbt:3"]) + // The same CTE, but each run's SQL is its own. + expect(queries[0].sql).not.toBe(queries[1].sql) + expect(queries[0].sql.replace(/^\/\* .*? \*\/\n/, "")).toBe(queries[1].sql.replace(/^\/\* .*? \*\/\n/, "")) + }) + + test("a call waiting on another call's start-up measurement stops at its own abort", async () => { + let finish = () => {} + const held = new Promise((resolve) => (finish = resolve)) + respond = async (sql) => { + if (sql === dbtCteProfileInternals.baselineQuery) await held + return { columnNames: ["_PROFILE_COUNT"], data: [{ _PROFILE_COUNT: 1 }] } + } + const tool = await initTool(DbtCteProfileTool) + const first = tool.execute({ model: "customers", cte: "orders" }, ctx) + while (baselines.length === 0) await sleep(1) + const controller = new AbortController() + const second = tool.execute({ model: "customers", cte: "final" }, { ...ctx, abort: controller.signal }) + // The second call's count has answered: it is now waiting on the first call's measurement. + while (queries.length < 2) await sleep(1) + await sleep(5) + expect(baselines).toHaveLength(1) + controller.abort(new Error("stopped")) + await expect(second).rejects.toThrow("stopped") + finish() + expect((await first).metadata.timing).toBe("baseline") + expect(baselines).toHaveLength(1) + }) + + test("a finished count waits only so long for a stalled start-up measurement", async () => { + respond = async (sql) => { + if (sql === dbtCteProfileInternals.baselineQuery) return new Promise(() => {}) + return { columnNames: ["_PROFILE_COUNT"], data: [{ _PROFILE_COUNT: 1 }] } + } + const saved = dbtCteProfileInternals.baselineWaitMs + dbtCteProfileInternals.baselineWaitMs = 20 + try { + const tool = await initTool(DbtCteProfileTool) + const result = await tool.execute({ model: "customers", cte: "orders" }, ctx) + expect(result.metadata.timing).toBe("wall") + expect(result.metadata.queryMs).toBe(result.metadata.totalMs) + } finally { + dbtCteProfileInternals.baselineWaitMs = saved + } + }) + + test("when the trivial query fails too, the call's own time is given as including start-up, and the next call measures again", async () => { + let fail = true + respond = async (sql) => { + if (sql === dbtCteProfileInternals.baselineQuery && fail) throw new Error("no warehouse") + return { columnNames: ["_PROFILE_COUNT"], data: [{ _PROFILE_COUNT: 1 }] } + } + const tool = await initTool(DbtCteProfileTool) + const first = await tool.execute({ model: "customers", cte: "orders" }, ctx) + expect(first.metadata).toMatchObject({ timing: "wall" }) + expect(first.metadata.queryMs).toBe(first.metadata.totalMs) + expect(first.output).toContain("start-up included") + fail = false + const second = await tool.execute({ model: "customers", cte: "orders" }, ctx) + expect(second.metadata.timing).toBe("baseline") + expect(baselines).toHaveLength(2) + }) +}) diff --git a/packages/opencode/test/altimate/dbt-health.test.ts b/packages/opencode/test/altimate/dbt-health.test.ts index 5cf8e83106..8277e4dc2b 100644 --- a/packages/opencode/test/altimate/dbt-health.test.ts +++ b/packages/opencode/test/altimate/dbt-health.test.ts @@ -1,7 +1,7 @@ // dbt_health: the DataPilot run through the project's dbt adapter. The adapter and the Python run are // replaced through their seams, so these tests need no dbt, no Python and no DataPilot. import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, test } from "bun:test" -import { mkdir, mkdtemp, realpath, rm, symlink, utimes, writeFile } from "node:fs/promises" +import { mkdir, mkdtemp, readFile, realpath, rm, symlink, utimes, writeFile } from "node:fs/promises" import fs from "node:fs" import os from "node:os" import path from "node:path" @@ -198,6 +198,28 @@ describe("dbt_health", () => { ]) }) + test("reads the project's catalog when the adapter's own dbt target folder has none", async () => { + // The tools' dbt writes to a folder of its own under target/, where `dbt docs generate` never puts a catalog. + const own = path.join(project, "target", ".altimate-code") + await mkdir(own, { recursive: true }) + await writeFile(path.join(own, "manifest.json"), await readFile(path.join(project, "target", "manifest.json"), "utf8")) + await writeFile(path.join(project, "target", "catalog.json"), "{}") + dbtAdapterInternals.create = async () => ({ + isInitialized: () => true, + getProjectRoot: () => project, + getManifestPath: () => path.join(own, "manifest.json"), + parseManifest: async () => parsed, + dispose: async () => {}, + }) + try { + await run() + } finally { + await rm(path.join(project, "target", "catalog.json")) + await rm(own, { recursive: true, force: true }) + } + expect(python.calls[0].args).toEqual([path.join(own, "manifest.json"), path.join(project, "target", "catalog.json"), ""]) + }) + test("runs DataPilot under the Python the dbt config names, not one found for the project", async () => { dbtAdapterInternals.readConfig = async () => ({ projectRoot: project, pythonPath: "/pyenv/dbt/bin/python", dbtIntegration: "corecommand", queryLimit: 500 }) await run() diff --git a/packages/opencode/test/altimate/dbt-lineage-adapter.test.ts b/packages/opencode/test/altimate/dbt-lineage-adapter.test.ts index 54c1b69d00..e616e5f982 100644 --- a/packages/opencode/test/altimate/dbt-lineage-adapter.test.ts +++ b/packages/opencode/test/altimate/dbt-lineage-adapter.test.ts @@ -435,12 +435,13 @@ describe("the dbt adapter sql_execute uses", () => { await disposeDbtAdapter() expect(disposed).toBe(1) - let finish!: (value: unknown) => void + let finish: ((value: unknown) => void) | undefined dbtAdapterInternals.create = () => new Promise((resolve) => (finish = resolve)) const starting = getDbtAdapter() - await new Promise((resolve) => setTimeout(resolve, 0)) + // Creation starts after the config and target-folder reads: wait until it is under way. + while (!finish) await new Promise((resolve) => setTimeout(resolve, 1)) await disposeDbtAdapter() - finish(adapter) + finish!(adapter) await starting await new Promise((resolve) => setTimeout(resolve, 0)) expect(disposed).toBe(2) diff --git a/packages/opencode/test/altimate/dbt-sql-graph.test.ts b/packages/opencode/test/altimate/dbt-sql-graph.test.ts index 58d6ae311e..7dff10d186 100644 --- a/packages/opencode/test/altimate/dbt-sql-graph.test.ts +++ b/packages/opencode/test/altimate/dbt-sql-graph.test.ts @@ -15,6 +15,8 @@ import { dbtAdapterInternals, resetDbtAdapter } from "@/altimate/native/connecti import { initTool } from "./tool-fixture" import { buildSqlGraph, + clauseName, + CLAUSE_NAME_CHARS, formatSqlGraph, mergeColumns, remapSqlLineage, @@ -625,7 +627,7 @@ describe("the backend's real node shapes", () => { expect(byId["raw.customers_222"]).toMatchObject({ nodeId: "source.shop.raw.customers", type: "source" }) expect(byId.LEFT_JOIN_9).toEqual({ id: "LEFT_JOIN_9", - name: "LEFT_JOIN_9", + name: "LEFT JOIN raw.customers ON 1 = 1", type: "LEFT_JOIN", sql: "LEFT JOIN raw.customers ON 1 = 1", columns: [], @@ -633,6 +635,13 @@ describe("the backend's real node shapes", () => { expect(byId.WHERE_7.sql).toBe("WHERE amount > 0") }) + test("a join or filter is named by its clause, not the backend's generated key, which stays its id", () => { + const byId = Object.fromEntries(graph.nodes.map((n) => [n.id, n])) + expect(byId.WHERE_7.id).toBe("WHERE_7") + expect(byId.WHERE_7.name).toBe("WHERE amount > 0") + expect(graph.edges).toContainEqual(["WHERE_7", "orders"]) + }) + test("two relations of one name and schema in different databases are told apart by the database", () => { const twoDbs: SqlLineageResponse = { tableEdges: [], @@ -845,3 +854,46 @@ describe("final-select identity and the text output", () => { expect(text).toContain(" sql: select id from t") }) }) + +describe("clauseName", () => { + // The shapes the backend sends for a dbt model's joins, filters and groups (DuckDB, quoted relations). + test.each([ + [{ type: "FILTER", expression: "WHERE stg_orders.status <> 'returned'" }, "WHERE stg_orders.status <> 'returned'"], + [{ type: "GROUP", expression: "GROUP BY\n items.order_id" }, "GROUP BY items.order_id"], + [ + { type: "INNER_JOIN", expression: 'JOIN "shop"."main"."stg_products" AS products ON items.product_id = products.product_id' }, + "INNER JOIN stg_products AS products ON items.product_id = products.product_id", + ], + [{ type: "LEFT_JOIN", expression: "LEFT JOIN payments AS payments ON orders.order_id = payments.order_id" }, "LEFT JOIN payments ON orders.order_id = payments.order_id"], + [{ type: "INNER_JOIN", expression: "JOIN `proj`.`ds`.`orders` AS `orders` ON a = b" }, "INNER JOIN orders ON a = b"], + [{ type: "LEFT_JOIN", expression: "LEFT JOIN raw.customers ON 1 = 1" }, "LEFT JOIN raw.customers ON 1 = 1"], + ])("%o is named %p", (details, name) => { + expect(clauseName(details)).toBe(name) + }) + + test("whitespace inside a quoted string or identifier is kept, and an alias differing only in case stays", () => { + expect(clauseName({ type: "FILTER", expression: "WHERE code = 'a, b''s'\n AND \"x y\" = 1" })).toBe("WHERE code = 'a, b''s' AND \"x y\" = 1") + expect(clauseName({ type: "LEFT_JOIN", expression: 'LEFT JOIN "db"."s"."Orders" AS "orders" ON 1 = 1' })).toBe("LEFT JOIN Orders AS orders ON 1 = 1") + expect(clauseName({ type: "FILTER", expression: "WHERE [order status]]x] = 1" })).toBe("WHERE [order status]]x] = 1") + expect(clauseName({ type: "FILTER", expression: "WHERE msg <> 'a\nb'" })).toBe("WHERE msg <> 'a b'") + // A name that needs its quotes keeps them. + expect(clauseName({ type: "LEFT_JOIN", expression: 'LEFT JOIN "db"."s"."my table" AS "x y" ON 1 = 1' })).toBe('LEFT JOIN "my table" AS "x y" ON 1 = 1') + }) + + test("an AS inside a condition is left as written", () => { + expect(clauseName({ type: "FILTER", expression: "WHERE CAST(a.b AS INT) > 0" })).toBe("WHERE CAST(a.b AS INT) > 0") + }) + + test("a long clause is cut with an ellipsis, and a node with no clause has no name of its own", () => { + const name = clauseName({ type: "FILTER", expression: `WHERE ${"x = 1 AND ".repeat(30)}y = 2` })! + expect(name.length).toBe(CLAUSE_NAME_CHARS) + expect(name.endsWith("…")).toBe(true) + expect(clauseName({ type: "GROUP" })).toBeUndefined() + const graph = remapSqlLineage( + { tableEdges: [], details: { Sort_1: { type: "SORT" } } }, + { name: "m", uniqueId: "model.p.m", resourceType: "model" }, + [], + ) + expect(graph.nodes.find((n) => n.id === "Sort_1")?.name).toBe("Sort_1") + }) +}) diff --git a/packages/opencode/test/altimate/impact-analysis.test.ts b/packages/opencode/test/altimate/impact-analysis.test.ts index 61845bbc15..a1638accd0 100644 --- a/packages/opencode/test/altimate/impact-analysis.test.ts +++ b/packages/opencode/test/altimate/impact-analysis.test.ts @@ -350,3 +350,29 @@ describe("impact_analysis: column reach carries how each column comes from the t } }) }) + +describe("impact_analysis: invalid arguments", () => { + // The CLI and MCP print this message as the answer: it has to say which field is wrong. + const message = async (args: unknown) => { + const tool = await initTool(ImpactAnalysisTool) + return Promise.resolve() + .then(() => tool.execute(args, ctx)) + .then( + () => "", + (error: unknown) => (error instanceof Error ? error.message : String(error)), + ) + } + + test("a missing change_type is named, with the values it takes", async () => { + const text = await message({ model: "stg_orders" }) + expect(text).toContain("invalid arguments: change_type is required (expected one of \"remove\"|\"rename\"|\"retype\"|\"add\"|\"modify\")") + expect(text).not.toContain("Declaration") + }) + + test("an unknown change_type and a wrong type are named field by field", async () => { + const text = await message({ model: 7, change_type: "drop" }) + expect(text).toContain("model: Invalid input: expected string, received number") + expect(text).toContain("change_type: Invalid option: expected one of") + expect(text).not.toContain("Declaration") + }) +}) diff --git a/packages/opencode/test/altimate/tool-cli/daemon.test.ts b/packages/opencode/test/altimate/tool-cli/daemon.test.ts index 00a4148ae8..d942edc716 100644 --- a/packages/opencode/test/altimate/tool-cli/daemon.test.ts +++ b/packages/opencode/test/altimate/tool-cli/daemon.test.ts @@ -140,6 +140,17 @@ describe("connection env in the socket name", () => { } }) + test("changes with the Python environment the dbt adapter runs dbt with", async () => { + const dir = await temp() + const none = socketName(dir, "1.0.0", undefined, undefined, { PATH: "/bin" }) + const venv = socketName(dir, "1.0.0", undefined, undefined, { PATH: "/bin", VIRTUAL_ENV: "/p/.venv" }) + // A daemon started outside the virtualenv found another Python: the caller inside it gets its own daemon. + expect(venv).not.toBe(none) + expect(socketName(dir, "1.0.0", undefined, undefined, { VIRTUAL_ENV: "/q/.venv" })).not.toBe(venv) + expect(socketName(dir, "1.0.0", undefined, undefined, { CONDA_PREFIX: "/opt/conda/envs/dbt" })).not.toBe(none) + expect(socketName(dir, "1.0.0", undefined, undefined, { VIRTUAL_ENV: "" })).toBe(none) + }) + test("uses the registry's prefix", async () => { const { CONN_ENV_PREFIX } = await import("../../../src/altimate/native/connections/registry") const dir = await temp()