diff --git a/packages/opencode/src/altimate/review/dbt-patterns.ts b/packages/opencode/src/altimate/review/dbt-patterns.ts index 83673d0609..98da929d49 100644 --- a/packages/opencode/src/altimate/review/dbt-patterns.ts +++ b/packages/opencode/src/altimate/review/dbt-patterns.ts @@ -926,6 +926,117 @@ const detectMultiCountDistinct: Detector = (c) => { ) } +const JOIN_TYPE_GAP = /^\s*(?:(?:natural|inner|left|right|full|cross)\s+)?(?:outer\s+)?$/i +const JOIN_CLAUSE_END = /^(?:join|where|group\s+by|order\s+by|qualify|having|union|limit|window|left|right|inner|full|cross|natural|lateral|select|from)\b|^;/i + +/** Normalised equality/comparison predicate: no whitespace, lowercase, `a=b` with sides sorted. */ +function predKey(p: string): string { + const t = p.replace(/\s+/g, "").toLowerCase() + const eq = /^([^=<>!]+)=([^=<>!]+)$/.exec(t) + return eq ? [eq[1], eq[2]].sort().join("=") : t +} + +/** + * Predicates in a JOIN's ON clause that an earlier INNER join of the same join chain already states + * and that don't reference the newly joined relation: the inner join already dropped every row they + * would reject, so they filter nothing. (After a LEFT join the same predicate is not redundant: the + * left join keeps the rows it rejects.) Literals are compared as written. Returns each such + * predicate as written (whitespace collapsed). + */ +export function redundantJoinPredicates(sql: string): string[] { + // Literal text stays (`= 'a'` and `= 'b'` differ); `and`/parens inside a literal are rare enough here. + const s = stripComments(sql).replace(/\{\{[\s\S]*?\}\}/g, "__jinja__").replace(/\{%[\s\S]*?%\}/g, " ") + const joins: { alias?: string; preds: { raw: string; key: string }[]; start: number; end: number; inner: boolean }[] = [] + for (const m of s.matchAll(/\bjoin\b/gi)) { + const inner = !/\b(left|right|full|cross|natural|outer)\s+(?:outer\s+)?$/i.test(s.slice(Math.max(0, m.index! - 20), m.index)) + let i = m.index! + 4 + const rest = s.slice(i) + let rel = /^\s*(\S+)/.exec(rest) + if (rest.trimStart().startsWith("(")) { + const open = i + rest.indexOf("(") + let depth = 0 + for (i = open; i < s.length; i++) { + if (s[i] === "(") depth++ + else if (s[i] === ")" && --depth === 0) break + } + i++ + rel = null + } else if (rel) i += rel[0].length + const tail = /^\s+(?:as\s+)?(\w+)\s+on\b/i.exec(s.slice(i)) ?? /^\s+on\b/i.exec(s.slice(i)) + if (!tail) { + joins.push({ preds: [], start: m.index!, end: i, inner }) + continue + } + const alias = tail[1] && tail[1].toLowerCase() !== "on" ? tail[1].toLowerCase() : rel?.[1].split(".").pop()?.toLowerCase() + let j = i + tail[0].length + const onStart = j + let depth = 0 + for (; j < s.length; j++) { + const ch = s[j] + if (ch === "(") depth++ + else if (ch === ")") { + if (depth === 0) break + depth-- + } else if (depth === 0 && /\W/.test(s[j - 1]) && JOIN_CLAUSE_END.test(s.slice(j, j + 12))) break + } + const clause = s.slice(onStart, j) + const preds = /\bor\b/i.test(clause) + ? [] + : topLevelAnd(clause).map((raw) => ({ raw: raw.replace(/\s+/g, " ").trim(), key: predKey(raw) })).filter((p) => p.key) + joins.push({ alias, preds, start: m.index!, end: j, inner }) + } + const out: string[] = [] + let seen: Set = new Set() + for (let k = 0; k < joins.length; k++) { + const prev = joins[k - 1] + const chained = prev && JOIN_TYPE_GAP.test(s.slice(prev.end, joins[k].start)) + if (!chained) seen = new Set() + const { alias, preds } = joins[k] + for (const p of preds) + if (chained && seen.has(p.key) && /=/.test(p.key) && !(alias && new RegExp(`\\b${alias}\\.`, "i").test(p.raw))) out.push(p.raw) + if (joins[k].inner) for (const p of preds) seen.add(p.key) + } + return out +} + +/** Split a predicate list on top-level AND. */ +function topLevelAnd(clause: string): string[] { + const parts: string[] = [] + let depth = 0 + let last = 0 + for (const m of clause.matchAll(/[()]|\band\b/gi)) { + if (m[0] === "(") depth++ + else if (m[0] === ")") depth-- + else if (depth === 0) { + parts.push(clause.slice(last, m.index)) + last = m.index! + m[0].length + } + } + parts.push(clause.slice(last)) + return parts.map((p) => p.trim()).filter(Boolean) +} + +// 46. A join predicate an earlier join of the same chain already states. +const detectRedundantJoinPredicate: Detector = (c) => { + if (!c.added.some((l) => /=/.test(l))) return null + const norm = (t: string) => stripComments(t).replace(/\s+/g, "").toLowerCase() + for (const pred of redundantJoinPredicates(c.newSql)) { + const key = norm(pred) + const hit = c.added.find((l) => norm(l).includes(key)) + if (!hit) continue + return pattern( + c, + "sql_quality", + "suggestion", + "redundant-join-predicate", + "join repeats a predicate an earlier join already applies", + `\`${pred}\` doesn't involve the relation this join adds and is already in an earlier join's ON clause, so it filters nothing. Remove it, or compare the joined relation's own column if that was the intent.`, + hit, + ) + } + return null +} + const MODEL_DETECTORS: Detector[] = [ detectClock, detectCrossJoin, @@ -960,6 +1071,7 @@ const MODEL_DETECTORS: Detector[] = [ detectBooleanPrecedence, detectOffsetNoOrder, detectMultiCountDistinct, + detectRedundantJoinPredicate, ] /** diff --git a/packages/opencode/src/altimate/review/orchestrate.ts b/packages/opencode/src/altimate/review/orchestrate.ts index f805e23f33..2449d45486 100644 --- a/packages/opencode/src/altimate/review/orchestrate.ts +++ b/packages/opencode/src/altimate/review/orchestrate.ts @@ -41,7 +41,23 @@ import { detectUnitTestFixtures } from "./unit-tests" import { finalSelectNames } from "./select-list" import { detectTestAdequacy, detectThresholdLoosening } from "./test-adequacy" import { incrementalFindings, snapshotFindings, snapshotYmlFindings } from "./incremental-semantics" -import { dbtProjectFindings, jinjaFindings, joinTargets, missingRefFindings, modelConfigFindings, packagesFindings, testsOnNullColumns } from "./project-rules" +import { + dbtProjectFindings, + dispatchKwargFindings, + dispatchVariants, + docRefFindings, + jinjaFindings, + joinTargets, + missingRefFindings, + modelConfigFindings, + isPackageProject, + packageLockFindings, + packagesFindings, + quickstartModelFindings, + testWindowFindings, + testsOnNullColumns, + yamlParseFindings, +} from "./project-rules" import { type AiContextFile, type AiLaneStatus, type AiReviewFile, type AiReviewInput, type AiReviewResult, safeReason } from "./ai-review" import { ruleTier } from "./rule-tiers" import { type AiVerifyEvidence, numberedSlice } from "./ai-verify" @@ -2187,6 +2203,18 @@ export async function runReview(input: OrchestrateInput): Promise__` variant in the repo doesn't take. + const dispatchVariantFindings = async (file: ChangedFile, newSql: string): Promise => { + if (!/\badapter\.dispatch\b/.test(newSql)) return [] + const texts = new Map([[file.path, newSql]]) + for (const name of new Set([...newSql.matchAll(/\badapter\.dispatch\s*\(\s*['"](?:\w+\.)?(\w+)['"]/g)].map((m) => m[1]))) { + const hits = (await input.grepProject?.(`__${name}(`).catch(() => [])) ?? [] + for (const path of new Set(hits.map((h) => h.path))) + if (!texts.has(path)) texts.set(path, (await getContent?.(path, "new").catch(() => undefined)) ?? "") + } + return dispatchKwargFindings(file, newSql, (name) => [...texts.values()].flatMap((t) => dispatchVariants(t, name))) + } + // Macros: the catalog rules that hold in a SQL template, and vars with no default. if (lanes.has("dbt_patterns")) { const macros = reviewable.filter((f) => f.kind === "macro" && f.status !== "deleted") @@ -2202,6 +2230,7 @@ export async function runReview(input: OrchestrateInput): Promise undefined), ]) if (file.kind === "project_config") return [...packagesFindings(file, newContent), ...dbtProjectFindings(file, oldContent, newContent)] - return jinjaFindings(file, oldContent, newContent) + return [...jinjaFindings(file, oldContent, newContent), ...(file.kind === "test" ? testWindowFindings(file) : [])] } catch (err) { failedFile(file, err) return [] @@ -2243,6 +2272,56 @@ export async function runReview(input: OrchestrateInput): Promise() + const checkedOther = (file: ChangedFile) => { + otherChecked.add(file.path) + touch([file]) + } + if (lanes.has("dbt_patterns")) { + const read = (p: string, side: "old" | "new") => getContent?.(p, side)?.catch(() => undefined) ?? Promise.resolve(undefined) + const ymls = reviewable.filter((f) => f.status !== "deleted" && /\.ya?ml$/i.test(f.path)) + const sets = await Promise.all( + ymls.map(async (file) => { + const [newContent, oldContent] = await Promise.all([ + read(file.path, "new"), + file.status === "added" ? Promise.resolve(undefined) : read(file.oldPath ?? file.path, "old"), + ]) + if (file.kind === "other") checkedOther(file) + return yamlParseFindings(file, oldContent, newContent) + }), + ) + for (const findings of sets) all.push(findings) + + for (const file of reviewable.filter((f) => f.status === "added" && /(^|\/)package-lock\.yml$/.test(f.path))) { + const dir = file.path.slice(0, file.path.length - "package-lock.yml".length) + const [projectYml, testsYml, nested] = await Promise.all([ + read(`${dir}dbt_project.yml`, "new"), + read(`${dir}integration_tests/dbt_project.yml`, "new"), + // `integration_tests//dbt_project.yml` (grepProject globs from the project root; + // a git pathspec `*` crosses directories). + (dir === (input.layout?.root ? input.layout.root + "/" : "") && input.grepProject?.("name", "integration_tests/*dbt_project.yml").then((h) => h ?? []).catch(() => [])) || + Promise.resolve([]), + ]) + if (projectYml === undefined && testsYml === undefined) continue + checkedOther(file) + all.push(packageLockFindings(file, isPackageProject(projectYml, !!testsYml || nested.length > 0))) + } + + const newModels = reviewable.filter((f) => f.kind === "model_sql" && f.status === "added") + if (newModels.length) { + const QUICKSTART = /(^|\/)\.quickstart\/quickstart\.ya?ml$/ + const changed = reviewable.find((f) => QUICKSTART.test(f.path) && f.status !== "deleted") + const qsPath = changed?.path ?? `${input.layout?.root ? input.layout.root + "/" : ""}.quickstart/quickstart.yml` + const content = await read(qsPath, "new") + if (content) { + if (changed) checkedOther(changed) + all.push(quickstartModelFindings(changed ?? newModels[0], qsPath, content, newModels.map((f) => modelNameFromPath(f.path)))) + } + } + } + // schema.yml-level detectors (test removal) — run on changed YAML files // regardless of tier, since deleting a guardrail test is always worth flagging. // Pass old + new content when available so the detector diffs structural YAML @@ -2264,6 +2343,22 @@ export async function runReview(input: OrchestrateInput): Promise>() + const docExists = (name: string) => { + if (!docLookups.has(name)) { + const block = new RegExp(`\\{%-?\\s*docs\\s+${name}\\s*-?%\\}`) + docLookups.set( + name, + (input.grepProject?.(`docs ${name}`, "*.md") ?? Promise.resolve([])) + // An unavailable search (undefined) is no evidence the block is missing. + .then((hits) => hits === undefined || hits.some((h) => block.test(h.text))) + // A failed search is no evidence the block is missing. + .catch(() => true), + ) + } + return docLookups.get(name)! + } const schemaFiles = reviewable.filter((f) => f.kind === "schema_yml") touch(schemaFiles) const sqlByModel = new Map([...ctxByPath.values()].filter((c) => c.newSql).map((c) => [modelNameFromPath(c.file.path).toLowerCase(), c.newSql!])) @@ -2282,6 +2377,7 @@ export async function runReview(input: OrchestrateInput): Promise f.kind === "other" || f.kind === "analysis") + .filter((f) => (f.kind === "other" && !otherChecked.has(f.path)) || f.kind === "analysis") .map((f) => ({ path: f.path, reason: "not a dbt model/macro/test/yml/seed file" })), ...reviewable .filter((f) => !touched.has(f.path)) diff --git a/packages/opencode/src/altimate/review/project-rules.ts b/packages/opencode/src/altimate/review/project-rules.ts index c78772a2d3..f198f2ded6 100644 --- a/packages/opencode/src/altimate/review/project-rules.ts +++ b/packages/opencode/src/altimate/review/project-rules.ts @@ -3,10 +3,11 @@ import path from "path" import YAML from "yaml" import { type Finding, type ReviewCategory, type Severity, makeFinding } from "./finding" -import { type ChangedFile, walkDiffLines } from "./diff-filter" +import { type ChangedFile, walkDiffLines, walkDiffRows } from "./diff-filter" import { incrementalFilter, inlineConfig } from "./incremental-semantics" import { parenBody, topLevel } from "./rule-catalog" import { capabilityFor } from "./adapter-capabilities" +import { ADAPTERS } from "./dbt-patterns" const nameOf = (p: string) => path.basename(p).replace(/\.(sql|py|ya?ml|csv)$/i, "") @@ -220,6 +221,34 @@ const macroSignatures = (sql: string | undefined) => { return out } +/** Each `{% macro name(…) %}` / `{% test name(…) %}` with its parameters (name and whether it has a default) + * and the head line range of its signature. Defaults may hold calls and lists. */ +function signatures(sql: string | undefined) { + const out = new Map() + const text = sql ?? "" + for (const m of text.matchAll(/\{%-?\s*(macro|test)\s+([\w.]+)\s*\(/g)) { + const open = m.index! + m[0].length - 1 + const body = parenBody(text, open) + const params: { name: string; hasDefault: boolean }[] = [] + let depth = 0 + let cur = "" + for (const ch of body + ",") { + if ("([{".includes(ch)) depth++ + else if (")]}".includes(ch)) depth-- + if (ch === "," && depth === 0) { + const name = /^\s*(\w+)/.exec(cur)?.[1] + if (name) params.push({ name, hasDefault: /=/.test(cur) }) + cur = "" + } else cur += ch + } + const first = text.slice(0, m.index).split("\n").length + out.set(m[2], { params, first, last: first + (m[0] + body).split("\n").length - 1, test: m[1] === "test" }) + } + return out +} + +const DISPATCH_VARIANT = new RegExp(`^(default|${ADAPTERS.join("|")})__\\w`, "i") + /** The comment text on a line (`-- …`, `/* … *\/`, `{# … #}`), or "". */ const commentOf = (l: string) => /--(.*)$|\/\*(.*?)(\*\/|$)|\{#(.*?)(#\}|$)/.exec(l)?.slice(1).filter(Boolean).join(" ") ?? "" const TEMP_MARKER = @@ -266,6 +295,34 @@ export function jinjaFindings(file: ChangedFile, oldContent: string | undefined, out.push(at ? { ...f, startLine: at.line, endLine: at.line } : f) } + // A parameter added to an existing macro with no default: every existing caller omits it, and Jinja + // binds an omitted parameter to Undefined, which is not `none` (an `is none` fallback never runs). + const oldSigs = signatures(oldContent) + for (const [name, sig] of signatures(newContent)) { + const old = oldSigs.get(name) + // An `__x` variant is called only by its dispatcher, which passes every argument. + if (!old || DISPATCH_VARIANT.test(name)) continue + const known = new Set(old.params.map((p) => p.name)) + for (const p of sig.params) { + // dbt itself passes `model` / `column_name` to a generic test. + if (p.hasDefault || known.has(p.name) || (sig.test && ["model", "column_name"].includes(p.name))) continue + const at = [...walkDiffLines(file.diff)].find( + (r) => r.kind === "+" && r.line >= sig.first && r.line <= sig.last && new RegExp(`\\b${p.name}\\b`).test(r.text), + ) + if (!at) continue + const f = finding( + file, + "macro-param-no-default", + "warning", + "sql_correctness", + `new parameter \`${p.name}\` of \`${name}\` has no default`, + `\`${p.name}\` was added to the existing ${sig.test ? "test" : "macro"} \`${name}\` without a default. Every call written before this change omits it, and Jinja passes an omitted parameter as Undefined, not \`none\`: an \`is none\` fallback is skipped and the value renders as an empty string. Give it a default (\`${p.name}=none\`) unless every caller is updated in this change.`, + at.text, + ) + out.push({ ...f, startLine: at.line, endLine: at.line }) + } + } + // A var() key renamed with no fallback: projects that set the old key lose their override silently. const oldVars = varNames(oldContent) const newVars = varNames(newContent) @@ -316,6 +373,272 @@ export function jinjaFindings(file: ChangedFile, oldContent: string | undefined, return out } +// ── YAML structure ────────────────────────────────────────────────────────── + +/** The first YAML error in `text` (all documents), with its 1-based line; undefined when it parses. */ +export function yamlError(text: string): { message: string; line?: number; duplicate: boolean } | undefined { + for (const doc of YAML.parseAllDocuments(text)) { + const e = doc.errors?.[0] + if (e) return { message: e.message.split("\n")[0].replace(/ at line \d+, column \d+:?$/, ""), line: e.linePos?.[0]?.line, duplicate: e.code === "DUPLICATE_KEY" } + } + return undefined +} + +/** A YAML file that parsed at base and no longer parses at head (a mis-indented key often + * becomes a duplicate key of the entry above it). */ +export function yamlParseFindings(file: ChangedFile, oldContent: string | undefined, newContent: string | undefined): Finding[] { + if (!/\.ya?ml$/i.test(file.path) || file.status === "deleted" || !newContent) return [] + const err = yamlError(newContent) + if (!err || (oldContent !== undefined && yamlError(oldContent))) return [] + const text = err.line ? newContent.split("\n")[err.line - 1] : undefined + const f = finding( + file, + "yaml-parse-error", + "warning", + "sql_correctness", + err.duplicate ? "YAML key defined twice" : "file no longer parses as YAML", + err.duplicate + ? `${err.message}${err.line ? ` (line ${err.line})` : ""}. A key indented one level too deep becomes a second entry of the mapping above it; parsers either reject the file or keep only one value. Check the indentation of the keys added here.` + : `${err.message}${err.line ? ` (line ${err.line})` : ""}. The file parsed before this change; tools that read it now fail.`, + text, + ) + return [err.line ? { ...f, startLine: err.line, endLine: err.line } : f] +} + +/** New models named like the public models a `.quickstart/quickstart.yml` lists, but missing from + * its `public_models`: the quickstart setup won't expose them. */ +export function quickstartModelFindings( + /** Where the finding goes: the quickstart file when the PR changes it, else the new model. */ + target: ChangedFile, + quickstartPath: string, + content: string | undefined, + newModels: string[], +): Finding[] { + // Lenient like PyYAML: a duplicate key elsewhere in the file must not hide the list. + let doc: any + try { + doc = YAML.parse(content ?? "", { uniqueKeys: false }) + } catch { + return [] + } + const listed: string[] = Array.isArray(doc?.public_models) ? doc.public_models.filter((x: unknown) => typeof x === "string") : [] + if (listed.length < 3) return [] + // The package prefix (`hubspot__…`) most public models share. + const counts = new Map() + for (const m of listed) { + const p = /^([a-z0-9_]+?)__/i.exec(m)?.[1] + if (p) counts.set(p, (counts.get(p) ?? 0) + 1) + } + const [prefix, n] = [...counts.entries()].sort((a, b) => b[1] - a[1])[0] ?? ["", 0] + if (!prefix || n < listed.length * 0.8) return [] + const have = new Set(listed.map((m) => m.toLowerCase())) + const missing = newModels.filter((m) => m.toLowerCase().startsWith(`${prefix.toLowerCase()}__`) && !have.has(m.toLowerCase())) + if (!missing.length) return [] + return [ + finding( + target, + "quickstart-public-model-missing", + "suggestion", + "materialization", + `new model${missing.length === 1 ? "" : "s"} missing from \`public_models\``, + `${missing.map((m) => `\`${m}\``).join(", ")} ${missing.length === 1 ? "is" : "are"} new and named like the ${listed.length} models \`${quickstartPath}\` lists in \`public_models\`, but ${missing.length === 1 ? "isn't" : "aren't"} listed. Add ${missing.length === 1 ? "it" : "them"}, or confirm ${missing.length === 1 ? "it is" : "they are"} meant to stay internal.`, + ), + ] +} + +// ── package-lock.yml ──────────────────────────────────────────────────────── + +/** True when the dbt project at `dir` is a package others install: it carries a dbt project + * under `integration_tests/` (at any depth), or its `dbt_project.yml` names no profile. */ +export function isPackageProject(projectYml: string | undefined, hasIntegrationTests: boolean): boolean { + if (hasIntegrationTests) return true + const doc = parseYaml(projectYml) + return !!doc && typeof doc === "object" && !doc.profile +} + +/** `package-lock.yml` added to a package: it pins the package's own dev dependencies, which + * projects installing the package never read, and drifts from `packages.yml` ranges. */ +export function packageLockFindings(file: ChangedFile, isPackage: boolean): Finding[] { + if (!/(^|\/)package-lock\.yml$/.test(file.path) || file.status !== "added" || !isPackage) return [] + return [ + finding( + file, + "package-lock-in-package", + "suggestion", + "materialization", + "`package-lock.yml` committed in a dbt package", + "This repository is a dbt package (it has an `integration_tests/` project or no `profile:`). Projects that install it resolve their own dependencies and never read this lock file, while it pins the package's development installs to versions that drift from the `packages.yml` ranges. Leave it out (add it to `.gitignore`) unless the package deliberately locks its CI.", + addedLines(file.diff)[0], + ), + ] +} + +// ── doc() references ──────────────────────────────────────────────────────── + +const DOC_REF = /\bdoc\s*\(\s*['"](\w+)['"]\s*\)/g + +/** `doc('x')` names this change adds to a yml file that no docs block in the project defines: + * dbt fails to parse the project. Only when another `doc()` in the file resolves in the project, + * so docs kept in an installed package aren't mistaken for missing ones. */ +export async function docRefFindings( + file: ChangedFile, + oldContent: string | undefined, + newContent: string | undefined, + docExists: (name: string) => Promise, +): Promise { + if (!newContent || file.status === "deleted") return [] + const before = new Set([...(oldContent ?? "").matchAll(DOC_REF)].map((m) => m[1])) + const lines = new Map() + for (const l of addedLines(file.diff)) for (const m of l.matchAll(DOC_REF)) if (!before.has(m[1]) && !lines.has(m[1])) lines.set(m[1], l) + if (!lines.size) return [] + const missing: string[] = [] + for (const name of [...lines.keys()].slice(0, 10)) if (!(await docExists(name))) missing.push(name) + if (!missing.length) return [] + const others = [...new Set([...newContent.matchAll(DOC_REF)].map((m) => m[1]))].filter((n) => !missing.includes(n)).slice(0, 5) + let resolves = false + for (const n of others) if ((resolves = await docExists(n))) break + if (!resolves) return [] + return missing.map((name) => + finding( + file, + "doc-block-missing", + "warning", + "sql_correctness", + `\`doc('${name}')\` has no docs block`, + `No \`{% docs ${name} %}\` block exists in the project's \`.md\` files, while the other \`doc()\` references in this file resolve there. dbt fails to parse the project on a \`doc()\` it can't resolve. Add the docs block, or fix the name.`, + lines.get(name), + ), + ) +} + +// ── singular test windows ─────────────────────────────────────────────────── + +const UNIT_SECONDS: Record = { second: 1, minute: 60, hour: 3600, day: 86400, week: 604800, month: 2592000, quarter: 7776000, year: 31536000 } +const UNIT = "(second|minute|hour|day|week|month|quarter|year)s?" +// `interval 1 day`, `interval '1' day`, `interval '30 minutes'`, `dateadd('day', -3, …)` / `dbt.dateadd(datepart='day', interval=-3, …)`. +const WINDOW_RES = [ + new RegExp(`\\binterval\\s+'?\\s*(\\d+)\\s*'?\\s*'?${UNIT}\\b'?`, "gi"), + new RegExp(`\\bdate_?add\\s*\\(\\s*(?:datepart\\s*=\\s*)?['"]?${UNIT}['"]?\\s*,\\s*(?:interval\\s*=\\s*)?-\\s*(\\d+)`, "gi"), +] + +/** Window lengths on a line in seconds, and the line with every window masked (for pairing edits). */ +function windows(line: string): { secs: number[]; shape: string } { + const secs: number[] = [] + let shape = line.replace(/--.*$/, "") + WINDOW_RES.forEach((re, i) => { + shape = shape.replace(re, (_m, a: string, b: string) => { + const [n, unit] = i === 0 ? [a, b] : [b, a] + secs.push(Number(n) * UNIT_SECONDS[unit.toLowerCase()]) + return "" + }) + }) + return { secs, shape: shape.replace(/\s+/g, " ").trim().toLowerCase() } +} + +/** A singular test whose lookback window literal got shorter: it now checks less data. */ +export function testWindowFindings(file: ChangedFile): Finding[] { + if (file.status !== "modified" || !/\.sql$/i.test(file.path)) return [] + const rows = [...walkDiffRows(file.diff)] + const removed = rows.filter((r) => r.kind === "-").map((r) => windows(r.text)).filter((w) => w.secs.length) + if (!removed.length) return [] + for (const r of rows) { + if (r.kind !== "+") continue + const now = windows(r.text) + if (!now.secs.length) continue + const was = removed.find((w) => w.shape === now.shape && w.secs.length === now.secs.length) + const shrunk = was && now.secs.some((s, i) => s < was.secs[i]) + if (!was || !shrunk) continue + const fmt = (s: number) => { + const [u, n] = Object.entries(UNIT_SECONDS).reverse().find(([, n]) => s % n === 0 && s >= n) ?? ["second", 1] + return `${s / n} ${u}${s / n === 1 ? "" : "s"}` + } + const i = now.secs.findIndex((s, k) => s < was.secs[k]) + const f = finding( + file, + "test-window-shrunk", + "suggestion", + "test_coverage", + "test lookback window got shorter", + `The window this test checks shrank from ${fmt(was.secs[i])} to ${fmt(now.secs[i])}, so rows outside the new window are no longer tested. If the test now runs on that schedule, say so; otherwise keep the longer window.`, + r.text, + ) + return [{ ...f, startLine: r.line, endLine: r.line }] + } + return [] +} + +// ── adapter.dispatch variants ─────────────────────────────────────────────── + +/** `adapter.dispatch('name', …)(…)` calls whose argument list carries `kw=` keywords on added + * lines: the dispatched macro name, and each such keyword with the raw added line. */ +export function addedDispatchKwargs(file: ChangedFile, sql: string): { name: string; kwargs: { kw: string; line: number; text: string }[] }[] { + const added = [...walkDiffLines(file.diff)].filter((r) => r.kind === "+") + const lineAt = (i: number) => sql.slice(0, i).split("\n").length + const out: { name: string; kwargs: { kw: string; line: number; text: string }[] }[] = [] + for (const m of sql.matchAll(/\badapter\.dispatch\s*\(\s*['"]([\w.]+)['"]/g)) { + const open = m.index! + m[0].indexOf("(") + const head = parenBody(sql, open) + const after = open + head.length + 2 + const call = /^\s*\(/.exec(sql.slice(after)) + if (!call) continue + const argsOpen = after + call[0].length - 1 + const args = parenBody(sql, argsOpen) + const first = lineAt(argsOpen) + const last = first + args.split("\n").length - 1 + const kws = [...topLevel(args).matchAll(/(?:^|,)\s*(\w+)\s*=(?!=)/g)].map((k) => k[1]) + const kwargs = kws.flatMap((kw) => { + const hit = added.find((r) => r.line >= first && r.line <= last && new RegExp(`\\b${kw}\\s*=(?!=)`).test(r.text)) + return hit ? [{ kw, line: hit.line, text: hit.text }] : [] + }) + if (kwargs.length) out.push({ name: m[1].split(".").pop()!, kwargs }) + } + return out +} + +/** Every `{% macro __(…) %}` in `text`: its parameters, and whether its body reads + * `kwargs` (Jinja then accepts any extra keyword). */ +export function dispatchVariants(text: string, name: string): { macro: string; params: string[]; acceptsKwargs: boolean }[] { + const out: { macro: string; params: string[]; acceptsKwargs: boolean }[] = [] + for (const [macro, sig] of signatures(text)) { + // `{% test %}` blocks are never dispatch targets. + if (sig.test || !macro.endsWith(`__${name}`) || macro === name) continue + const at = new RegExp(`\\{%-?\\s*macro\\s+${escapeRe(macro)}\\s*\\(`).exec(text)?.index + if (at === undefined) continue + const end = text.slice(at).search(/\{%-?\s*endmacro\b/) + const body = text.slice(at, end < 0 ? undefined : at + end) + out.push({ macro, params: sig.params.map((p) => p.name), acceptsKwargs: /\bkwargs\b/.test(body) || /\bvarargs\b/.test(body) }) + } + return out +} + +/** Keywords a changed `adapter.dispatch` call now passes that an in-repo variant does not take: + * on that adapter dbt fails with "takes no keyword argument". */ +export function dispatchKwargFindings( + file: ChangedFile, + sql: string, + variantsOf: (name: string) => { macro: string; params: string[]; acceptsKwargs: boolean }[], +): Finding[] { + const out: Finding[] = [] + for (const call of addedDispatchKwargs(file, sql)) { + const variants = variantsOf(call.name) + for (const { kw, line, text } of call.kwargs) { + const missing = variants.filter((v) => !v.acceptsKwargs && !v.params.includes(kw)).map((v) => v.macro) + if (!missing.length) continue + const f = finding( + file, + "dispatch-kwarg-not-in-variant", + "warning", + "sql_correctness", + `\`${call.name}\` now passes \`${kw}\`, which ${missing.length === 1 ? "a variant doesn't" : "some variants don't"} accept`, + `The \`adapter.dispatch('${call.name}')\` call passes \`${kw}=\`, but ${missing.map((v) => `\`${v}\``).join(", ")} ${missing.length === 1 ? "has" : "have"} no \`${kw}\` parameter. On ${missing.length === 1 ? "that adapter" : "those adapters"} dbt fails with "takes no keyword argument '${kw}'". Add the parameter (with a default) to every variant.`, + text, + ) + out.push({ ...f, startLine: line, endLine: line }) + } + } + return out +} + // ── model config combinations ─────────────────────────────────────────────── const unquote = (v: string | undefined) => (v ?? "").trim().replace(/^['"]|['"]$/g, "") @@ -353,7 +676,7 @@ export function modelConfigFindings( if (!/config\s*\(|unique_key|partition_by|batch_size|materialized|incremental|is_incremental/i.test(touched)) return [] const out: Finding[] = [] const materialized = (unquote(cfg.materialized) || String(manifestConfig?.materialized ?? "")).toLowerCase() - const strategy = unquote(cfg.incremental_strategy).toLowerCase() + const strategy = (unquote(cfg.incremental_strategy) || String(manifestConfig?.incremental_strategy ?? "")).toLowerCase() if (cfg.unique_key && (materialized === "table" || materialized === "view")) out.push( @@ -411,6 +734,23 @@ export function modelConfigFindings( ), ) } + // BigQuery merges scan the whole target unless `incremental_predicates` bound it: the merge + // condition alone never prunes target partitions. (`merge` is BigQuery's default strategy.) + const bigquery = dialectKnown ? /bigquery/i.test(dialect) : /['"]field['"]\s*:/.test(cfg.partition_by ?? "") + const predicates = cfg.incremental_predicates ?? manifestConfig?.incremental_predicates + const hasPredicates = Array.isArray(predicates) ? predicates.length > 0 : !!predicates && !/^\s*(none|\[\s*\])\s*$/i.test(String(predicates)) + if (bigquery && parts.length && uk.length && (strategy === "merge" || !strategy) && !hasPredicates) + out.push( + finding( + file, + "merge-no-incremental-predicates", + "suggestion", + "warehouse_cost", + "incremental merge has no `incremental_predicates`", + `This model merges on \`unique_key\` into a table partitioned by ${parts.map((p) => `\`${p}\``).join(", ")}, with no \`incremental_predicates\`. BigQuery can't prune the target from the merge condition, so every run scans the whole table. Add a predicate on the partition column, e.g. \`incremental_predicates=["DBT_INTERNAL_DEST.${parts[0]} >= "]\`.`, + addedLine(file, /\b(unique_key|partition_by|incremental_strategy)\b/) ?? addedLine(file, /\bis_incremental\s*\(/), + ), + ) // The incremental filter should constrain the partition column, or each run scans every partition. const filter = incrementalFilter(newSql) if (parts.length && filter && !parts.some((p) => new RegExp(`\\b${p}\\b`, "i").test(filter))) diff --git a/packages/opencode/src/altimate/review/rule-catalog.ts b/packages/opencode/src/altimate/review/rule-catalog.ts index 8769b1d980..ef66a6e4bb 100644 --- a/packages/opencode/src/altimate/review/rule-catalog.ts +++ b/packages/opencode/src/altimate/review/rule-catalog.ts @@ -21,7 +21,7 @@ import { capabilityFor, type AdapterCapability } from "./adapter-capabilities" * Redshift/Databricks anti-pattern docs, r/dataengineering, dbt-core issues. */ -export type RuleKind = "model_sql" | "schema_yml" | "snapshot" | "any" +export type RuleKind = "model_sql" | "schema_yml" | "snapshot" | "macro" | "any" export interface RuleExample { add?: string // text treated as added (+) lines @@ -416,7 +416,13 @@ export function evaluateCatalog( /** Build a RuleInput from an example (for the catalog self-test). */ export function inputFromExample(ex: RuleExample, kind: RuleKind = "model_sql"): RuleInput { const defaultPath = - kind === "schema_yml" ? "models/marts/_models.yml" : kind === "snapshot" ? "snapshots/s.sql" : "models/marts/m.sql" + kind === "schema_yml" + ? "models/marts/_models.yml" + : kind === "snapshot" + ? "snapshots/s.sql" + : kind === "macro" + ? "macros/m.sql" + : "models/marts/m.sql" const file: ChangedFile = { path: ex.path ?? defaultPath, status: "modified" } const added = ex.add ? ex.add.split("\n") : [] const removed = ex.remove ? ex.remove.split("\n") : [] @@ -777,6 +783,35 @@ const CORRECTNESS: Rule[] = [ { add: "group by 1, 2" }, { add: "group by customer_id" }, ), + r( + "jinja-concat-missing-space", + "sql_correctness", + "suggestion", + "SQL text concatenated in Jinja without a space", + "A string of SQL is joined with `~` to a macro call or variable with no space at the seam, so the rendered SQL runs the two together (`ticket_id` + `, source_relation` works, `ticket_id` + `source_relation` does not). Put a space inside the string next to `~`.", + { + // A literal holding SQL (it has whitespace) that ends, or starts, with a word character right at the `~`, + // inside an open `{{`/`{%` on the line (a bare `col ~ 'a b'` is a Postgres regex match, not Jinja). + added: + /\{[{%](?:(?![}%]\}).)*?(?:'[^'\n]*\s[^'\n]*[A-Za-z0-9]'\s*~\s*[A-Za-z_][\w.]*|[\w)]\s*~\s*'[A-Za-z0-9][^'\n]*\s[^'\n]*')/, + // Messages are not SQL. + unless: /\b(?:log|print|raise_compiler_error|warn)\s*\(/, + macros: true, + }, + { add: "{{ dbt.datediff('a', 'lead(a) over (partition by ticket_id' ~ pkg.partition_cols() ~ ' order by a)', 'second') }}" }, + { add: "{{ dbt.datediff('a', 'lead(a) over (partition by ticket_id ' ~ pkg.partition_cols() ~ ' order by a)', 'second') }}" }, + ), + r( + "macro-group-by-ordinal", + "sql_quality", + "suggestion", + "GROUP BY a column position in macro SQL", + "A macro's SELECT list is usually built from arguments or loops, so the column at that position depends on the caller. `GROUP BY ` then groups by whichever column lands there. Group by the column name or expression.", + // A single position; two or more are `group-by-positional`. + { added: /\bgroup\s+by\s+\d+\b(?!\s*,)/i, kind: "macro" }, + { add: " SELECT max(t.loaded_at) AS loaded_at, 'src' AS src_label\n FROM t\n GROUP BY 2" }, + { add: " SELECT max(t.loaded_at) AS loaded_at, 'src' AS src_label\n FROM t\n GROUP BY src_label" }, + ), r( "order-by-positional", "sql_quality", diff --git a/packages/opencode/src/altimate/review/rule-tiers.ts b/packages/opencode/src/altimate/review/rule-tiers.ts index 0262a833dc..efb701ae5d 100644 --- a/packages/opencode/src/altimate/review/rule-tiers.ts +++ b/packages/opencode/src/altimate/review/rule-tiers.ts @@ -117,6 +117,18 @@ const SUMMARY = [ "rule-catalog:union-not-all", "rule-catalog:unqualified-column-multi-table", "rule-catalog:where-1-eq-1", + // Macro, test and yml rules: summary until each is judged on replayed PRs. + "rule-catalog:macro-group-by-ordinal", + "dbt-project-rules:macro-param-no-default", + "dbt-project-rules:dispatch-kwarg-not-in-variant", + "dbt-project-rules:test-window-shrunk", + "dbt-project-rules:yaml-parse-error", + "dbt-project-rules:quickstart-public-model-missing", + "dbt-project-rules:doc-block-missing", + "dbt-project-rules:package-lock-in-package", + "dbt-project-rules:merge-no-incremental-predicates", + "dbt-patterns:redundant-join-predicate", + "rule-catalog:jinja-concat-missing-space", ] export const DEFAULT_RULE_TIERS: Record = { diff --git a/packages/opencode/test/altimate/review-dbt-patterns.test.ts b/packages/opencode/test/altimate/review-dbt-patterns.test.ts index 2dd4a14751..75e6058b3a 100644 --- a/packages/opencode/test/altimate/review-dbt-patterns.test.ts +++ b/packages/opencode/test/altimate/review-dbt-patterns.test.ts @@ -1605,3 +1605,53 @@ describe("dbt-patterns extended battery", () => { test("safe_divide is NOT flagged", () => expect(has(fire("x", ["select safe_divide(a, b) as r"]), "sql_correctness")).toBe(false)) }) + +describe("join predicate repeated from an earlier join", () => { + const sql = (extra: string) => + [ + "select e.member_id, s.score", + "from {{ ref('accounts') }} as e", + "inner join", + " {{ ref('expected') }} as a", + " on e.region = a.target_region", + " and e.level = a.target_level", + "inner join", + " scores as s", + " on a.target_metric = s.metric", + ` ${extra}`, + ].join("\n") + const redundant = (body: string, added: string[]) => + detectModelPatterns(modelFile("models/m.sql", body, added), body, DEFAULT_RUBRIC).filter((f) => f.rule === "dbt-patterns:redundant-join-predicate") + test("a later join restating an earlier join's predicate: fires on that added line", () => { + const body = sql("and a.target_region = e.region") + const fs = redundant(body, [" and a.target_region = e.region"]) + expect(fs).toHaveLength(1) + expect((fs[0].evidence?.result as { line?: string }).line?.trim()).toBe("and a.target_region = e.region") + }) + test("twin: a predicate on the newly joined relation is silent", () => { + const body = sql("and s.region = e.region") + expect(redundant(body, [" and s.region = e.region"])).toHaveLength(0) + }) + test("after a LEFT JOIN the repeated predicate still filters: silent", () => { + const body = sql("and a.target_region = e.region").replace("inner join\n {{ ref('expected') }}", "left join\n {{ ref('expected') }}") + expect(redundant(body, [" and a.target_region = e.region"])).toHaveLength(0) + }) + test("predicates comparing to different literals are different predicates", () => { + const body = [ + "select * from p", + "inner join a on a.id = p.id and p.platform = 'web'", + "inner join b on b.id = p.id and p.platform = 'app'", + ].join("\n") + expect(redundant(body, ["inner join b on b.id = p.id and p.platform = 'app'"])).toHaveLength(0) + const same = body.replace("'app'", "'web'") + expect(redundant(same, ["inner join b on b.id = p.id and p.platform = 'web'"])).toHaveLength(1) + }) + test("the same predicate in a separate query (another CTE) is not the same join chain", () => { + const body = [ + "with x as (select * from e join a on e.region = a.target_region),", + "y as (select * from e join s on e.id = s.id where 1 = 1)", + "select * from y join a on e.region = a.target_region", + ].join("\n") + expect(redundant(body, ["select * from y join a on e.region = a.target_region"])).toHaveLength(0) + }) +}) diff --git a/packages/opencode/test/altimate/review-project-rules.test.ts b/packages/opencode/test/altimate/review-project-rules.test.ts index 2b97a04bf1..41db3f998e 100644 --- a/packages/opencode/test/altimate/review-project-rules.test.ts +++ b/packages/opencode/test/altimate/review-project-rules.test.ts @@ -1,17 +1,25 @@ import { describe, test, expect } from "bun:test" import { dbtProjectFindings, + dispatchKwargFindings, + dispatchVariants, + docRefFindings, jinjaBalance, jinjaFindings, joinTargets, literalNullColumns, missingRefFindings, modelConfigFindings, + packageLockFindings, packagesFindings, + quickstartModelFindings, + testWindowFindings, testsOnNullColumns, + yamlParseFindings, } from "../../src/altimate/review/project-rules" import { CATALOG, ruleMatches, leftJoinNullifiedByWhere, evaluateCatalog } from "../../src/altimate/review/rule-catalog" -import { isSuppressiblePortability } from "../../src/altimate/review/orchestrate" +import { inlineConfigSplats, isSuppressiblePortability, runReview, type ReviewRunner } from "../../src/altimate/review/orchestrate" +import { DEFAULT_REVIEW_CONFIG } from "../../src/altimate/review/config" import { detectMacroPatterns } from "../../src/altimate/review/dbt-patterns" import { DEFAULT_RUBRIC } from "../../src/altimate/review/rubric" import type { ChangedFile } from "../../src/altimate/review/diff-filter" @@ -339,3 +347,353 @@ describe("config read from Jinja variables and project defaults", () => { expect(titles(modelConfigFindings(file("models/m.sql", sql), sql, "bigquery", { materialized: "incremental" }))).not.toContain("m: `unique_key` has no effect on a table") }) }) + +describe("GROUP BY a single position in macro SQL", () => { + const macro = (body: string, path = "macros/tables/hub.sql") => + detectMacroPatterns({ path, status: "modified", diff: plus(body) }, body, DEFAULT_RUBRIC, new Set()) + const ordinal = (fs: T[]) => fs.filter((f) => f.rule === "rule-catalog:macro-group-by-ordinal") + const body = "{% macro hub(src_loaded_at) %}\nSELECT max(t.{{ src_loaded_at }}) AS {{ src_loaded_at }},\n'x' AS src_label\nFROM {{ this }} t\nGROUP BY 2\n{% endmacro %}" + test("fires on the added GROUP BY line and records it", () => { + const hits = ordinal(macro(body)) + expect(hits).toHaveLength(1) + expect((hits[0].evidence?.result as { line?: string }).line).toBe("GROUP BY 2") + }) + test("twin: grouping by the column name is silent", () => { + expect(ordinal(macro(body.replace("GROUP BY 2", "GROUP BY src_label")))).toHaveLength(0) + }) + test("two positions stay with group-by-positional; a model's GROUP BY 1 is not this rule", () => { + expect(ordinal(macro(body.replace("GROUP BY 2", "GROUP BY 1, 2")))).toHaveLength(0) + const sql = "select a, count(*) from t\ngroup by 1" + const model = evaluateCatalog({ path: "models/m.sql", status: "modified" }, sql, sql.split("\n"), [], DEFAULT_RUBRIC) + expect(ordinal(model)).toHaveLength(0) + }) +}) + +describe("parameter added to an existing macro without a default", () => { + const noDefault = (diff: string, old: string, head: string, path = "macros/tests/anomalies.sql") => + jinjaFindings({ path, status: "modified", diff }, old, head).filter((f) => f.rule === "dbt-project-rules:macro-param-no-default") + const old = "{% test anomalies(\n model,\n days_back,\n sensitivity=3\n) %}\nselect 1\n{% endtest %}" + const head = (p: string) => `{% test anomalies(\n model,\n days_back,\n ${p},\n sensitivity=3\n) %}\nselect 1\n{% endtest %}` + const diff = (p: string) => `@@ -2,3 +2,4 @@\n model,\n days_back,\n+ ${p},\n sensitivity=3\n` + test("fires on the added parameter line of a test block", () => { + const fs = noDefault(diff("floor_value"), old, head("floor_value")) + expect(fs).toHaveLength(1) + expect(fs[0].startLine).toBe(4) + expect((fs[0].evidence!.result as { line?: string }).line).toBe(" floor_value,") + }) + test("twin: the same parameter with `=none` is silent", () => { + expect(noDefault(diff("floor_value=none"), old, head("floor_value=none"))).toHaveLength(0) + }) + test("a new macro, or a parameter that already existed, is not a new required parameter", () => { + expect(jinjaFindings({ path: "macros/m.sql", status: "added", diff: plus(head("floor_value")) }, undefined, head("floor_value")).filter((f) => f.rule === "dbt-project-rules:macro-param-no-default")).toHaveLength(0) + const renamedOnly = "@@ -1 +1 @@\n-{% macro m(a, b) %}\n+{% macro m(a, b) -%}\n" + expect(noDefault(renamedOnly, "{% macro m(a, b) %}{% endmacro %}", "{% macro m(a, b) -%}{% endmacro %}", "macros/m.sql")).toHaveLength(0) + }) + test("an adapter variant gets every argument from its dispatcher: silent", () => { + const o = "{% macro snowflake__sat(a, b) %}{% endmacro %}" + const h = "{% macro snowflake__sat(a, b, skip_hwm) %}{% endmacro %}" + expect(noDefault(`@@ -1 +1 @@\n-${o}\n+${h}\n`, o, h, "macros/sat.sql")).toHaveLength(0) + const g = "{% macro sat(a, b, skip_hwm) %}{% endmacro %}" + expect(noDefault(`@@ -1 +1 @@\n-${o.replace("snowflake__", "")}\n+${g}\n`, o.replace("snowflake__", ""), g, "macros/sat.sql")).toHaveLength(1) + }) + test("defaults holding calls and lists do not split the parameter list", () => { + const o = "{% macro m(a, cols=['x', 'y'], ts=dbt.current_timestamp()) %}{% endmacro %}" + const h = "{% macro m(a, cols=['x', 'y'], ts=dbt.current_timestamp(), b) %}{% endmacro %}" + const fs = noDefault(`@@ -1 +1 @@\n-${o}\n+${h}\n`, o, h, "macros/m.sql") + expect(fs.map((f) => f.title)).toEqual(["m: new parameter `b` of `m` has no default"]) + }) +}) + +describe("adapter.dispatch keyword a variant does not take", () => { + const head = (variantParams: string, body = "select 1") => + [ + "{%- macro ghost(col, dtype, size=none, scale=none) -%}", + "{{ return(adapter.dispatch('ghost', 'pkg')(col=col,", + " dtype=dtype,", + " size=size,", + " scale=scale)) }}", + "{%- endmacro -%}", + "", + "{%- macro default__ghost(col, dtype, size, scale=none) -%}select 1{%- endmacro -%}", + `{%- macro snowflake__ghost(${variantParams}) -%}${body}{%- endmacro -%}`, + ].join("\n") + const diff = "@@ -3,2 +3,3 @@\n dtype=dtype,\n size=size,\n+ scale=scale)) }}\n" + const run = (sql: string) => { + const f: ChangedFile = { path: "macros/ghost.sql", status: "modified", diff } + return dispatchKwargFindings(f, sql, (name) => dispatchVariants(sql, name)) + } + test("a variant without the new keyword: fires on the added keyword line", () => { + const fs = run(head("col, dtype, size")) + expect(fs).toHaveLength(1) + expect(fs[0].rule).toBe("dbt-project-rules:dispatch-kwarg-not-in-variant") + expect(fs[0].startLine).toBe(5) + expect(fs[0].body).toContain("`snowflake__ghost`") + expect(fs[0].body).not.toContain("`default__ghost`") + }) + test("twin: every variant takes the keyword", () => { + expect(run(head("col, dtype, size, scale=none"))).toHaveLength(0) + }) + test("a variant that reads `kwargs` accepts any keyword", () => { + expect(run(head("col, dtype, size", "{{ kwargs.get('scale') }}"))).toHaveLength(0) + }) + test("a `{% test %}` block named like a variant is skipped, not a crash", () => { + const sql = head("col, dtype, size, scale=none") + "\n{% test bigquery__ghost(model) %}select 1{% endtest %}" + expect(dispatchVariants(sql, "ghost").map((v) => v.macro)).toEqual(["default__ghost", "snowflake__ghost"]) + }) + test("keywords on unchanged lines are not this change", () => { + const f: ChangedFile = { path: "macros/ghost.sql", status: "modified", diff: "@@ -9 +9 @@\n-x\n+y\n" } + const sql = head("col, dtype, size") + expect(dispatchKwargFindings(f, sql, (name) => dispatchVariants(sql, name))).toHaveLength(0) + }) +}) + +describe("singular test lookback window shortened", () => { + const edit = (before: string, after: string) => + testWindowFindings({ path: "tests/recent_rows.sql", status: "modified", diff: `@@ -5,3 +5,3 @@\n from t\n-${before}\n+${after}\n group by 1\n` }) + test("1 day to 30 minutes: fires on the added line", () => { + const fs = edit( + "where event_at >= timestamp_sub(current_timestamp(), interval 1 day)", + "where event_at >= timestamp_sub(current_timestamp(), interval 30 minute)", + ) + expect(fs).toHaveLength(1) + expect(fs[0].startLine).toBe(6) + expect(fs[0].body).toContain("from 1 day to 30 minutes") + }) + test("dbt.dateadd with a negative day count", () => { + expect(edit("where d >= {{ dbt.dateadd('day', -7, 'current_date') }}", "where d >= {{ dbt.dateadd('day', -3, 'current_date') }}")).toHaveLength(1) + }) + test("twin: the same window written another way, or a longer window, is silent", () => { + expect(edit("where d >= current_date - interval 1 day", "where d >= current_date - interval 24 hour")).toHaveLength(0) + expect(edit("where d >= current_date - interval 1 day", "where d >= current_date - interval 2 day")).toHaveLength(0) + }) + test("a rewritten filter (window replaced by a var) is not a literal shrink", () => { + expect(edit("where d >= timestamp_sub(ts, interval 1 hour)", "where d >= timestamp('{{ var(\"batch_start\") }}')")).toHaveLength(0) + }) +}) + +/** A whole review over files given as head/base contents (diff = every head line added). */ +async function prReview( + files: { path: string; status?: ChangedFile["status"]; head?: string; base?: string; diff?: string }[], + extra: Partial[0]> & { tree?: Record } = {}, +) { + const runner: ReviewRunner = { + async impact() { + return { hasManifest: true, severity: "SAFE", directCount: 0, transitiveCount: 0, testCount: 0 } + }, + async grade() { + return { grade: "A" } + }, + async check() { + return { issues: [], ran: true } + }, + async equivalence() { + return { decided: true, equivalent: true, differences: [], confidence: "high" } as any + }, + async detectPii() { + return { columns: [] } + }, + } + const byPath = new Map(files.map((f) => [f.path, f])) + const { tree = {}, ...rest } = extra + return runReview({ + changedFiles: files.map((f) => ({ path: f.path, status: f.status ?? "modified", diff: f.diff ?? plus(f.head ?? "") })), + config: { ...DEFAULT_REVIEW_CONFIG, reviewers: ["dbt_patterns"] }, + rubric: DEFAULT_RUBRIC, + mode: "comment", + runner, + forceTier: "full", + getContent: async (p, side) => (side === "old" ? byPath.get(p)?.base : (byPath.get(p)?.head ?? tree[p])), + ...rest, + }) +} +const ruleIds = (fs: { rule?: string }[]) => fs.map((f) => f.rule) + +describe("YAML that stops parsing", () => { + const base = "supported_vars:\n order_metadata:\n type: dictionary\n display_name: Customer\n" + const good = base + "\n refund_metadata:\n type: dictionary\n display_name: Charge\n" + const bad = base + "\n refund_metadata:\n type: dictionary\n display_name: Charge\n" + const f = (head: string): ChangedFile => ({ path: ".quickstart/quickstart.yml", status: "modified", diff: `@@ -4,0 +5,4 @@\n+\n+${head.split("\n").slice(5, 8).join("\n+")}\n` }) + test("a key indented under the previous entry duplicates its keys: fires on the error line", () => { + const fs = yamlParseFindings(f(bad), base, bad) + expect(fs).toHaveLength(1) + expect(fs[0].rule).toBe("dbt-project-rules:yaml-parse-error") + expect(fs[0].title).toContain("YAML key defined twice") + expect(fs[0].startLine).toBe(6) + expect((fs[0].evidence!.result as { line?: string }).line).toBe(" refund_metadata:") + }) + test("twin: the same key at the right indentation is silent", () => { + expect(yamlParseFindings(f(good), base, good)).toHaveLength(0) + }) + test("a file that already failed at base is not this change", () => { + expect(yamlParseFindings(f(bad), bad, bad)).toHaveLength(0) + }) + test("runs on a quickstart file the review otherwise doesn't classify, and lists it as reviewed", async () => { + const env = await prReview([{ path: ".quickstart/quickstart.yml", head: bad, base }]) + expect(ruleIds(env.findings)).toContain("dbt-project-rules:yaml-parse-error") + expect((env.scope?.notReviewed ?? []).map((n) => n.path)).not.toContain(".quickstart/quickstart.yml") + }) +}) + +describe("new model missing from the quickstart public_models", () => { + const qs = 'public_models: [\n "crm__deals",\n "crm__contacts",\n "crm__companies"\n]\n' + const model = "select 1 as id" + test("a new crm__ model not listed: fires on the quickstart file", async () => { + const env = await prReview([ + { path: ".quickstart/quickstart.yml", head: qs + "# touched\n", base: qs }, + { path: "models/history/crm__daily_deal_history.sql", status: "added", head: model }, + { path: "models/intermediate/int_crm__deal_spine.sql", status: "added", head: model }, + ]) + const f = env.findings.find((x) => x.rule === "dbt-project-rules:quickstart-public-model-missing")! + expect(f.file).toBe(".quickstart/quickstart.yml") + expect(f.body).toContain("`crm__daily_deal_history`") + expect(f.body).not.toContain("int_crm__deal_spine") + }) + test("twin: the new model is listed", async () => { + const listed = qs.replace('"crm__companies"', '"crm__companies",\n "crm__daily_deal_history"') + const env = await prReview([ + { path: ".quickstart/quickstart.yml", head: listed, base: qs }, + { path: "models/history/crm__daily_deal_history.sql", status: "added", head: model }, + ]) + expect(ruleIds(env.findings)).not.toContain("dbt-project-rules:quickstart-public-model-missing") + }) + test("an unrelated duplicate key elsewhere in the quickstart file doesn't hide the list", () => { + const dup = qs + "table_variables:\n crm_enabled:\n - a\n crm_enabled:\n - b\n" + const m: ChangedFile = { path: "models/crm__new.sql", status: "added" } + expect(quickstartModelFindings(m, ".quickstart/quickstart.yml", dup, ["crm__new"])).toHaveLength(1) + expect(quickstartModelFindings(m, ".quickstart/quickstart.yml", dup, ["crm__deals"])).toHaveLength(0) + }) + test("quickstart unchanged: the finding goes on the new model", () => { + const m: ChangedFile = { path: "models/crm__new.sql", status: "added" } + expect(quickstartModelFindings(m, ".quickstart/quickstart.yml", qs, ["crm__new"])[0].file).toBe("models/crm__new.sql") + expect(quickstartModelFindings(m, ".quickstart/quickstart.yml", qs, ["other__new"])).toHaveLength(0) + }) +}) + +describe("doc() reference with no docs block", () => { + const base = "models:\n - name: stg_ads\n columns:\n - name: campaign_id\n description: \"{{ doc('campaign_id') }}\"\n" + const head = (name: string) => base + ` - name: ${name}\n description: "{{ doc('${name}') }}"\n` + const yml = (name: string): ChangedFile => ({ + path: "models/staging/stg_ads.yml", + status: "modified", + diff: `@@ -5,0 +6,2 @@\n+ - name: ${name}\n+ description: "{{ doc('${name}') }}"\n`, + }) + const defined = new Set(["campaign_id", "user_id"]) + const exists = async (n: string) => defined.has(n) + test("a new doc() name no docs block defines: fires on its added line", async () => { + const fs = await docRefFindings(yml("audience_id"), base, head("audience_id"), exists) + expect(fs).toHaveLength(1) + expect(fs[0].rule).toBe("dbt-project-rules:doc-block-missing") + expect((fs[0].evidence!.result as { line?: string }).line).toBe(` description: "{{ doc('audience_id') }}"`) + }) + test("twin: a doc() name the project defines is silent", async () => { + expect(await docRefFindings(yml("user_id"), base, head("user_id"), exists)).toHaveLength(0) + }) + test("no other doc() in the file resolves locally (docs may come from a package): silent", async () => { + expect(await docRefFindings(yml("audience_id"), base, head("audience_id"), async () => false)).toHaveLength(0) + }) + test("end to end: the project's .md files are searched for the block", async () => { + const searched: string[] = [] + const env = await prReview([{ path: "models/staging/stg_ads.yml", head: head("audience_id"), base, diff: yml("audience_id").diff }], { + grepProject: async (pattern, glob) => { + searched.push(`${pattern} ${glob}`) + return pattern === "docs campaign_id" ? [{ path: "models/docs.md", line: 3, text: "{% docs campaign_id %}" }] : [] + }, + }) + expect(ruleIds(env.findings)).toContain("dbt-project-rules:doc-block-missing") + expect(searched).toContain("docs audience_id *.md") + }) +}) + +describe("package-lock.yml added to a dbt package", () => { + const lock = "packages:\n - package: dbt-labs/dbt_utils\n version: 1.3.0\nsha1_hash: abc\n" + const run = (tree: Record) => + prReview([{ path: "package-lock.yml", status: "added", head: lock }], { tree }) + test("a package (integration_tests project present): fires on the file's first line", async () => { + const env = await run({ "dbt_project.yml": "name: pkg\nprofile: dev\n", "integration_tests/dbt_project.yml": "name: pkg_tests\nprofile: ci\n" }) + const f = env.findings.find((x) => x.rule === "dbt-project-rules:package-lock-in-package")! + expect(f.file).toBe("package-lock.yml") + expect((f.evidence!.result as { line?: string }).line).toBe("packages:") + expect((env.scope?.notReviewed ?? []).map((n) => n.path)).not.toContain("package-lock.yml") + }) + test("a package whose integration project is nested under integration_tests//", async () => { + const env = await prReview([{ path: "package-lock.yml", status: "added", head: lock }], { + tree: { "dbt_project.yml": "name: pkg\nprofile: dev\n" }, + grepProject: async (_p, glob) => (glob === "integration_tests/*dbt_project.yml" ? [{ path: "integration_tests/project/dbt_project.yml", line: 1, text: "name: pkg_tests" }] : []), + }) + expect(ruleIds(env.findings)).toContain("dbt-project-rules:package-lock-in-package") + }) + test("a package with no profile and no integration tests also counts", async () => { + expect(ruleIds((await run({ "dbt_project.yml": "name: pkg\n" })).findings)).toContain("dbt-project-rules:package-lock-in-package") + }) + test("twin: an application project (profile, no integration_tests) commits its lock file", async () => { + expect(ruleIds((await run({ "dbt_project.yml": "name: app\nprofile: warehouse\n" })).findings)).not.toContain("dbt-project-rules:package-lock-in-package") + }) + test("a lock file that was already committed (modified) is not this change", () => { + expect(packageLockFindings({ path: "package-lock.yml", status: "modified", diff: plus(lock) }, true)).toHaveLength(0) + }) +}) + +describe("BigQuery incremental merge without incremental_predicates", () => { + const sql = (extra = "") => + [ + "{% set meta_config = {", + ' "materialized": "incremental",', + ' "unique_key": ["event_id"],', + ' "partition_by": {"field": "event_at", "data_type": "timestamp", "granularity": "month"}' + extra, + "} %}", + "{{ config(**meta_config) }}", + "select * from {{ ref('ops') }}", + "{% if is_incremental() %}", + "where event_at >= timestamp('{{ var(\"batch_start\") }}')", + "{% endif %}", + ].join("\n") + const predicates = (body: string, dialect = "bigquery") => + modelConfigFindings( + { path: "models/marts/ops.sql", status: "modified", diff: "@@ -8,2 +8,3 @@\n+{% if is_incremental() %}\n+where event_at >= timestamp('{{ var(\"batch_start\") }}')\n" }, + inlineConfigSplats(body), + dialect, + ).filter((f) => f.rule === "dbt-project-rules:merge-no-incremental-predicates") + test("partitioned merge with a unique_key and no predicates, incremental filter changed: fires", () => { + const fs = predicates(sql()) + expect(fs).toHaveLength(1) + expect(fs[0].body).toContain("`event_at`") + }) + test("a strategy set in dbt_project.yml (manifest config) counts", () => { + const run = (strategy: string) => + modelConfigFindings( + { path: "models/marts/ops.sql", status: "modified", diff: "@@ -8,2 +8,3 @@\n+{% if is_incremental() %}\n+where event_at >= timestamp('{{ var(\"batch_start\") }}')\n" }, + inlineConfigSplats(sql()), + "bigquery", + { incremental_strategy: strategy }, + ).filter((f) => f.rule === "dbt-project-rules:merge-no-incremental-predicates") + expect(run("insert_overwrite")).toHaveLength(0) + expect(run("merge")).toHaveLength(1) + }) + test("twin: incremental_predicates set", () => { + expect(predicates(sql(',\n "incremental_predicates": ["DBT_INTERNAL_DEST.event_at >= current_date - 3"]'))).toHaveLength(0) + }) + test("another engine is left to partition-not-in-merge-key", () => { + expect(predicates(sql(), "snowflake")).toHaveLength(0) + }) +}) + +describe("SQL text concatenated in Jinja without a space", () => { + const seam = (line: string) => fires("jinja-concat-missing-space", line) + test("a SQL literal ending in a word right at `~`", () => { + expect(seam("{{ dbt.datediff('changed_at', 'coalesce(lead(changed_at) over (partition by ticket_id' ~ pkg.cols() ~ ' order by changed_at), now()', 'second') }}")).toBe(true) + expect(seam("{{ 'select id from' ~ relation }}")).toBe(true) + }) + test("twin: the space is inside the string", () => { + expect(seam("{{ dbt.datediff('changed_at', 'coalesce(lead(changed_at) over (partition by ticket_id ' ~ pkg.cols() ~ ' order by changed_at), now()', 'second') }}")).toBe(false) + }) + test("identifier building is not SQL text", () => { + expect(seam("{{ ref('stg_' ~ source_name) }}")).toBe(false) + expect(seam("{% set rel = target.schema ~ '_staging' %}")).toBe(false) + expect(seam("{{ \"'\" ~ value ~ \"'\" }}")).toBe(false) + }) + test("a Postgres regex match is not Jinja; messages are not SQL", () => { + expect(seam("where name ~ 'acme corp'")).toBe(false) + expect(seam("where {{ col }} ~ 'acme corp' and x = {{ var('y') }}")).toBe(false) + expect(seam("{{ log('Running model' ~ model.name, info=true) }}")).toBe(false) + }) +})