Repository navigation
feat(review): 11 deterministic rules for macro, test and yml defects #1770
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: feat/review-ai-verifier
Are you sure you want to change the base?
Changes from all commits
76e1f46
692a3d9
176ced1
7f8e34b
00b3d26
6b3f0ec
758fa57
48a5f3c
e62a13f
4526dd2
5667da5
936b24a
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change | ||||||
|---|---|---|---|---|---|---|---|---|
|
|
@@ -926,6 +926,117 @@ const detectMultiCountDistinct: Detector = (c) => { | |||||||
| ) | ||||||||
| } | ||||||||
|
|
||||||||
| const JOIN_TYPE_GAP = /^\s*(?:(?:natural|inner|left|right|full|cross)\s+)?(?:outer\s+)?$/i | ||||||||
| const JOIN_CLAUSE_END = /^(?:join|where|group\s+by|order\s+by|qualify|having|union|limit|window|left|right|inner|full|cross|natural|lateral|select|from)\b|^;/i | ||||||||
|
|
||||||||
| /** Normalised equality/comparison predicate: no whitespace, lowercase, `a=b` with sides sorted. */ | ||||||||
| function predKey(p: string): string { | ||||||||
| const t = p.replace(/\s+/g, "").toLowerCase() | ||||||||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. P2: Prompt for AI agents |
||||||||
| const eq = /^([^=<>!]+)=([^=<>!]+)$/.exec(t) | ||||||||
| return eq ? [eq[1], eq[2]].sort().join("=") : t | ||||||||
| } | ||||||||
|
|
||||||||
| /** | ||||||||
| * Predicates in a JOIN's ON clause that an earlier INNER join of the same join chain already states | ||||||||
| * and that don't reference the newly joined relation: the inner join already dropped every row they | ||||||||
| * would reject, so they filter nothing. (After a LEFT join the same predicate is not redundant: the | ||||||||
| * left join keeps the rows it rejects.) Literals are compared as written. Returns each such | ||||||||
| * predicate as written (whitespace collapsed). | ||||||||
| */ | ||||||||
| export function redundantJoinPredicates(sql: string): string[] { | ||||||||
| // Literal text stays (`= 'a'` and `= 'b'` differ); `and`/parens inside a literal are rare enough here. | ||||||||
| const s = stripComments(sql).replace(/\{\{[\s\S]*?\}\}/g, "__jinja__").replace(/\{%[\s\S]*?%\}/g, " ") | ||||||||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. SUGGESTION: literal text now feeds the structural join scan
|
||||||||
| const joins: { alias?: string; preds: { raw: string; key: string }[]; start: number; end: number; inner: boolean }[] = [] | ||||||||
| for (const m of s.matchAll(/\bjoin\b/gi)) { | ||||||||
| const inner = !/\b(left|right|full|cross|natural|outer)\s+(?:outer\s+)?$/i.test(s.slice(Math.max(0, m.index! - 20), m.index)) | ||||||||
| let i = m.index! + 4 | ||||||||
| const rest = s.slice(i) | ||||||||
| let rel = /^\s*(\S+)/.exec(rest) | ||||||||
| if (rest.trimStart().startsWith("(")) { | ||||||||
| const open = i + rest.indexOf("(") | ||||||||
| let depth = 0 | ||||||||
| for (i = open; i < s.length; i++) { | ||||||||
| if (s[i] === "(") depth++ | ||||||||
| else if (s[i] === ")" && --depth === 0) break | ||||||||
| } | ||||||||
| i++ | ||||||||
| rel = null | ||||||||
| } else if (rel) i += rel[0].length | ||||||||
| const tail = /^\s+(?:as\s+)?(\w+)\s+on\b/i.exec(s.slice(i)) ?? /^\s+on\b/i.exec(s.slice(i)) | ||||||||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. P3: The join parser accepts only unquoted word aliases, so Prompt for AI agents |
||||||||
| if (!tail) { | ||||||||
| joins.push({ preds: [], start: m.index!, end: i, inner }) | ||||||||
| continue | ||||||||
| } | ||||||||
| const alias = tail[1] && tail[1].toLowerCase() !== "on" ? tail[1].toLowerCase() : rel?.[1].split(".").pop()?.toLowerCase() | ||||||||
| let j = i + tail[0].length | ||||||||
| const onStart = j | ||||||||
| let depth = 0 | ||||||||
| for (; j < s.length; j++) { | ||||||||
| const ch = s[j] | ||||||||
| if (ch === "(") depth++ | ||||||||
| else if (ch === ")") { | ||||||||
| if (depth === 0) break | ||||||||
| depth-- | ||||||||
| } else if (depth === 0 && /\W/.test(s[j - 1]) && JOIN_CLAUSE_END.test(s.slice(j, j + 12))) break | ||||||||
| } | ||||||||
| const clause = s.slice(onStart, j) | ||||||||
| const preds = /\bor\b/i.test(clause) | ||||||||
| ? [] | ||||||||
| : topLevelAnd(clause).map((raw) => ({ raw: raw.replace(/\s+/g, " ").trim(), key: predKey(raw) })).filter((p) => p.key) | ||||||||
| joins.push({ alias, preds, start: m.index!, end: j, inner }) | ||||||||
| } | ||||||||
| const out: string[] = [] | ||||||||
| let seen: Set<string> = new Set() | ||||||||
| for (let k = 0; k < joins.length; k++) { | ||||||||
| const prev = joins[k - 1] | ||||||||
| const chained = prev && JOIN_TYPE_GAP.test(s.slice(prev.end, joins[k].start)) | ||||||||
| if (!chained) seen = new Set() | ||||||||
| const { alias, preds } = joins[k] | ||||||||
| for (const p of preds) | ||||||||
| if (chained && seen.has(p.key) && /=/.test(p.key) && !(alias && new RegExp(`\\b${alias}\\.`, "i").test(p.raw))) out.push(p.raw) | ||||||||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. [MAJOR] Unescaped join relation text used in !(alias && new RegExp(`\\b${alias}\\.`, "i").test(p.raw))When a join has no alias, Fix: use
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. [MINOR] Predicates stay in if (joins[k].inner) for (const p of preds) seen.add(p.key)
Fix: clear |
||||||||
| if (joins[k].inner) for (const p of preds) seen.add(p.key) | ||||||||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. P2: Prompt for AI agents
Suggested change
|
||||||||
| } | ||||||||
| return out | ||||||||
| } | ||||||||
|
|
||||||||
| /** Split a predicate list on top-level AND. */ | ||||||||
| function topLevelAnd(clause: string): string[] { | ||||||||
| const parts: string[] = [] | ||||||||
| let depth = 0 | ||||||||
| let last = 0 | ||||||||
| for (const m of clause.matchAll(/[()]|\band\b/gi)) { | ||||||||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. P2: Prompt for AI agentsThere was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. SUGGESTION: splitting on
|
||||||||
| if (m[0] === "(") depth++ | ||||||||
| else if (m[0] === ")") depth-- | ||||||||
| else if (depth === 0) { | ||||||||
| parts.push(clause.slice(last, m.index)) | ||||||||
| last = m.index! + m[0].length | ||||||||
| } | ||||||||
| } | ||||||||
| parts.push(clause.slice(last)) | ||||||||
| return parts.map((p) => p.trim()).filter(Boolean) | ||||||||
| } | ||||||||
|
|
||||||||
| // 46. A join predicate an earlier join of the same chain already states. | ||||||||
| const detectRedundantJoinPredicate: Detector = (c) => { | ||||||||
| if (!c.added.some((l) => /=/.test(l))) return null | ||||||||
| const norm = (t: string) => stripComments(t).replace(/\s+/g, "").toLowerCase() | ||||||||
| for (const pred of redundantJoinPredicates(c.newSql)) { | ||||||||
| const key = norm(pred) | ||||||||
| const hit = c.added.find((l) => norm(l).includes(key)) | ||||||||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. WARNING: the anchor search can point at the legitimate earlier join
|
||||||||
| if (!hit) continue | ||||||||
| return pattern( | ||||||||
| c, | ||||||||
| "sql_quality", | ||||||||
| "suggestion", | ||||||||
| "redundant-join-predicate", | ||||||||
| "join repeats a predicate an earlier join already applies", | ||||||||
| `\`${pred}\` doesn't involve the relation this join adds and is already in an earlier join's ON clause, so it filters nothing. Remove it, or compare the joined relation's own column if that was the intent.`, | ||||||||
| hit, | ||||||||
| ) | ||||||||
| } | ||||||||
| return null | ||||||||
| } | ||||||||
|
|
||||||||
| const MODEL_DETECTORS: Detector[] = [ | ||||||||
| detectClock, | ||||||||
| detectCrossJoin, | ||||||||
|
|
@@ -960,6 +1071,7 @@ const MODEL_DETECTORS: Detector[] = [ | |||||||
| detectBooleanPrecedence, | ||||||||
| detectOffsetNoOrder, | ||||||||
| detectMultiCountDistinct, | ||||||||
| detectRedundantJoinPredicate, | ||||||||
| ] | ||||||||
|
|
||||||||
| /** | ||||||||
|
|
||||||||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -41,7 +41,23 @@ import { detectUnitTestFixtures } from "./unit-tests" | |
| import { finalSelectNames } from "./select-list" | ||
| import { detectTestAdequacy, detectThresholdLoosening } from "./test-adequacy" | ||
| import { incrementalFindings, snapshotFindings, snapshotYmlFindings } from "./incremental-semantics" | ||
| import { dbtProjectFindings, jinjaFindings, joinTargets, missingRefFindings, modelConfigFindings, packagesFindings, testsOnNullColumns } from "./project-rules" | ||
| import { | ||
| dbtProjectFindings, | ||
| dispatchKwargFindings, | ||
| dispatchVariants, | ||
| docRefFindings, | ||
| jinjaFindings, | ||
| joinTargets, | ||
| missingRefFindings, | ||
| modelConfigFindings, | ||
| isPackageProject, | ||
| packageLockFindings, | ||
| packagesFindings, | ||
| quickstartModelFindings, | ||
| testWindowFindings, | ||
| testsOnNullColumns, | ||
| yamlParseFindings, | ||
| } from "./project-rules" | ||
| import { type AiContextFile, type AiLaneStatus, type AiReviewFile, type AiReviewInput, type AiReviewResult, safeReason } from "./ai-review" | ||
| import { ruleTier } from "./rule-tiers" | ||
| import { type AiVerifyEvidence, numberedSlice } from "./ai-verify" | ||
|
|
@@ -2187,6 +2203,18 @@ export async function runReview(input: OrchestrateInput): Promise<VerdictEnvelop | |
| for (const findings of sets) all.push(findings) | ||
| } | ||
|
|
||
| // An `adapter.dispatch` call that now passes a keyword some `<adapter>__` variant in the repo doesn't take. | ||
| const dispatchVariantFindings = async (file: ChangedFile, newSql: string): Promise<Finding[]> => { | ||
| if (!/\badapter\.dispatch\b/.test(newSql)) return [] | ||
| const texts = new Map<string, string>([[file.path, newSql]]) | ||
| for (const name of new Set([...newSql.matchAll(/\badapter\.dispatch\s*\(\s*['"](?:\w+\.)?(\w+)['"]/g)].map((m) => m[1]))) { | ||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. P2: Prompt for AI agents |
||
| const hits = (await input.grepProject?.(`__${name}(`).catch(() => [])) ?? [] | ||
| for (const path of new Set(hits.map((h) => h.path))) | ||
| if (!texts.has(path)) texts.set(path, (await getContent?.(path, "new").catch(() => undefined)) ?? "") | ||
| } | ||
| return dispatchKwargFindings(file, newSql, (name) => [...texts.values()].flatMap((t) => dispatchVariants(t, name))) | ||
| } | ||
|
|
||
| // Macros: the catalog rules that hold in a SQL template, and vars with no default. | ||
| if (lanes.has("dbt_patterns")) { | ||
| const macros = reviewable.filter((f) => f.kind === "macro" && f.status !== "deleted") | ||
|
|
@@ -2202,6 +2230,7 @@ export async function runReview(input: OrchestrateInput): Promise<VerdictEnvelop | |
| return [ | ||
| ...detectMacroPatterns(file, newSql, input.rubric, projectVars, { dialect, dialectKnown: !!input.dialectKnown }), | ||
| ...jinjaFindings(file, oldSql, newSql), | ||
| ...(await dispatchVariantFindings(file, newSql)), | ||
| ] | ||
| } catch (err) { | ||
| failedFile(file, err) | ||
|
|
@@ -2233,7 +2262,7 @@ export async function runReview(input: OrchestrateInput): Promise<VerdictEnvelop | |
| file.status === "added" ? Promise.resolve(undefined) : getContent?.(file.oldPath ?? file.path, "old")?.catch(() => undefined), | ||
| ]) | ||
| if (file.kind === "project_config") return [...packagesFindings(file, newContent), ...dbtProjectFindings(file, oldContent, newContent)] | ||
| return jinjaFindings(file, oldContent, newContent) | ||
| return [...jinjaFindings(file, oldContent, newContent), ...(file.kind === "test" ? testWindowFindings(file) : [])] | ||
| } catch (err) { | ||
| failedFile(file, err) | ||
| return [] | ||
|
|
@@ -2243,6 +2272,56 @@ export async function runReview(input: OrchestrateInput): Promise<VerdictEnvelop | |
| for (const findings of sets) all.push(findings) | ||
| } | ||
|
|
||
| // Files checked whatever their dbt role (a `.quickstart/quickstart.yml` is "other"): YAML that | ||
| // no longer parses, and new models missing from the quickstart `public_models`. | ||
| const otherChecked = new Set<string>() | ||
| const checkedOther = (file: ChangedFile) => { | ||
| otherChecked.add(file.path) | ||
| touch([file]) | ||
| } | ||
| if (lanes.has("dbt_patterns")) { | ||
| const read = (p: string, side: "old" | "new") => getContent?.(p, side)?.catch(() => undefined) ?? Promise.resolve(undefined) | ||
| const ymls = reviewable.filter((f) => f.status !== "deleted" && /\.ya?ml$/i.test(f.path)) | ||
| const sets = await Promise.all( | ||
| ymls.map(async (file) => { | ||
| const [newContent, oldContent] = await Promise.all([ | ||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. SUGGESTION: every changed yml now costs two extra
|
||
| read(file.path, "new"), | ||
| file.status === "added" ? Promise.resolve(undefined) : read(file.oldPath ?? file.path, "old"), | ||
| ]) | ||
| if (file.kind === "other") checkedOther(file) | ||
| return yamlParseFindings(file, oldContent, newContent) | ||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. P3: For added YAML files, Prompt for AI agents |
||
| }), | ||
| ) | ||
| for (const findings of sets) all.push(findings) | ||
|
|
||
| for (const file of reviewable.filter((f) => f.status === "added" && /(^|\/)package-lock\.yml$/.test(f.path))) { | ||
| const dir = file.path.slice(0, file.path.length - "package-lock.yml".length) | ||
| const [projectYml, testsYml, nested] = await Promise.all([ | ||
| read(`${dir}dbt_project.yml`, "new"), | ||
| read(`${dir}integration_tests/dbt_project.yml`, "new"), | ||
| // `integration_tests/<name>/dbt_project.yml` (grepProject globs from the project root; | ||
| // a git pathspec `*` crosses directories). | ||
| (dir === (input.layout?.root ? input.layout.root + "/" : "") && input.grepProject?.("name", "integration_tests/*dbt_project.yml").then((h) => h ?? []).catch(() => [])) || | ||
| Promise.resolve([]), | ||
| ]) | ||
| if (projectYml === undefined && testsYml === undefined) continue | ||
| checkedOther(file) | ||
| all.push(packageLockFindings(file, isPackageProject(projectYml, !!testsYml || nested.length > 0))) | ||
| } | ||
|
|
||
| const newModels = reviewable.filter((f) => f.kind === "model_sql" && f.status === "added") | ||
| if (newModels.length) { | ||
| const QUICKSTART = /(^|\/)\.quickstart\/quickstart\.ya?ml$/ | ||
| const changed = reviewable.find((f) => QUICKSTART.test(f.path) && f.status !== "deleted") | ||
| const qsPath = changed?.path ?? `${input.layout?.root ? input.layout.root + "/" : ""}.quickstart/quickstart.yml` | ||
| const content = await read(qsPath, "new") | ||
| if (content) { | ||
| if (changed) checkedOther(changed) | ||
| all.push(quickstartModelFindings(changed ?? newModels[0], qsPath, content, newModels.map((f) => modelNameFromPath(f.path)))) | ||
| } | ||
| } | ||
| } | ||
|
|
||
| // schema.yml-level detectors (test removal) — run on changed YAML files | ||
| // regardless of tier, since deleting a guardrail test is always worth flagging. | ||
| // Pass old + new content when available so the detector diffs structural YAML | ||
|
|
@@ -2264,6 +2343,22 @@ export async function runReview(input: OrchestrateInput): Promise<VerdictEnvelop | |
| // | ||
| // Fetches run in parallel across schema files (a schema-heavy PR could touch | ||
| // dozens of yml files; serial `git show` per file adds up). | ||
| // A `{% docs name %}` block in any of the project's .md files at head (cached per name). | ||
| const docLookups = new Map<string, Promise<boolean>>() | ||
| const docExists = (name: string) => { | ||
| if (!docLookups.has(name)) { | ||
| const block = new RegExp(`\\{%-?\\s*docs\\s+${name}\\s*-?%\\}`) | ||
| docLookups.set( | ||
| name, | ||
| (input.grepProject?.(`docs ${name}`, "*.md") ?? Promise.resolve([])) | ||
| // An unavailable search (undefined) is no evidence the block is missing. | ||
| .then((hits) => hits === undefined || hits.some((h) => block.test(h.text))) | ||
| // A failed search is no evidence the block is missing. | ||
| .catch(() => true), | ||
| ) | ||
| } | ||
| return docLookups.get(name)! | ||
| } | ||
| const schemaFiles = reviewable.filter((f) => f.kind === "schema_yml") | ||
| touch(schemaFiles) | ||
| const sqlByModel = new Map([...ctxByPath.values()].filter((c) => c.newSql).map((c) => [modelNameFromPath(c.file.path).toLowerCase(), c.newSql!])) | ||
|
|
@@ -2282,6 +2377,7 @@ export async function runReview(input: OrchestrateInput): Promise<VerdictEnvelop | |
| ...detectUnitTestFixtures(file, oldContent, newContent), | ||
| ...snapshotYmlFindings(file, oldContent, newContent), | ||
| ...detectTestAdequacy({ file, oldContent, newContent, projectYml, layout: input.layout }), | ||
| ...(lanes.has("dbt_patterns") && input.grepProject ? await docRefFindings(file, oldContent, newContent, docExists) : []), | ||
| ] | ||
| } catch (err) { | ||
| // One malformed schema file degrades that file only. | ||
|
|
@@ -2523,7 +2619,7 @@ export async function runReview(input: OrchestrateInput): Promise<VerdictEnvelop | |
| ...(input.scope?.notReviewed ?? []), | ||
| ...dropped, | ||
| ...reviewable | ||
| .filter((f) => f.kind === "other" || f.kind === "analysis") | ||
| .filter((f) => (f.kind === "other" && !otherChecked.has(f.path)) || f.kind === "analysis") | ||
| .map((f) => ({ path: f.path, reason: "not a dbt model/macro/test/yml/seed file" })), | ||
| ...reviewable | ||
| .filter((f) => !touched.has(f.path)) | ||
|
|
||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
[MINOR]
predKeylowercases and strips whitespace inside string literals, so different predicates compare equalfrom a inner join b on a.flag = 'X' inner join c on b.id = c.id and a.flag = 'x'flagsa.flag = 'x'as redundant (reproduced), but with'X'filtering first it is not: removing it as suggested changes results. The doc comment says "Literals are compared as written", which this contradicts.Fix: normalise only outside quoted literals, keep literal contents exact.