Repository navigation
feat(review): opt-in test-gap check and focused AI batches; verifier reads every changed file #1772
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: feat/review-ai-input-coverage
Are you sure you want to change the base?
Changes from all commits
6d2363b
591e49e
0650473
6a16b4a
86b289d
f695be5
31cb5c4
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Large diffs are not rendered by default.
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -41,6 +41,8 @@ export interface AiVerifyEvidence { | |
| prBody?: string | ||
| /** Head-side line numbers each changed file adds (see `addedLinesByFile`). */ | ||
| addedLines: Map<string, Set<number>> | ||
| /** Every file the PR changes, repo-relative, including tests and CI outside the dbt project. */ | ||
| prFiles?: string[] | ||
| } | ||
|
|
||
| export type Decision = "keep" | "discard" | "summary" | ||
|
|
@@ -379,12 +381,27 @@ export function hunkAround(diff: string | undefined, line?: number): string { | |
| : text || "(the line is outside the diff)" | ||
| } | ||
|
|
||
| /** Most PR file paths listed in a verifier message. */ | ||
| const PR_FILES_MAX = 40 | ||
|
|
||
| /** The user message for one batch: the PR title and each candidate with its hunk. Not the whole file, not the grounding. */ | ||
| export function buildVerifyMessage( | ||
| prTitle: string | undefined, | ||
| candidates: Array<{ id: string; finding: Finding; diff?: string }>, | ||
| prFiles: string[] = [], | ||
| ): string { | ||
| const parts = [`PR: ${(prTitle ?? "(untitled)").slice(0, 200)}`, "", "Candidates:"] | ||
| const parts = [`PR: ${(prTitle ?? "(untitled)").slice(0, 200)}`] | ||
| // A test-coverage claim is checked against the PR's own test files, which may sit outside the dbt project. | ||
| if (prFiles.length && candidates.some((c) => c.finding.category === "test_coverage")) { | ||
| const shown = prFiles.slice(0, PR_FILES_MAX) | ||
| parts.push( | ||
| "", | ||
| "Files this PR changes (read_file can open any of them):", | ||
| ...shown.map((p) => `- ${p}`), | ||
| ...(prFiles.length > shown.length ? [`- … and ${prFiles.length - shown.length} more`] : []), | ||
| ) | ||
| } | ||
| parts.push("", "Candidates:") | ||
| for (const { id, finding: f, diff } of candidates) { | ||
| parts.push( | ||
| "", | ||
|
|
@@ -764,6 +781,7 @@ export function verifierTools( | |
| ...ev.addedLines.keys(), | ||
| ...graph.parents.map((p) => p.path), | ||
| ...graph.downstream.map((d) => d.path), | ||
| ...(ev.prFiles ?? []), | ||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. P2: The verifier's read allowlist now includes every PR file, including paths the user excluded with Prompt for AI agents |
||
| ]), | ||
| ] | ||
| /** A requested path inside the review scope, as the repo-relative path to read; a project-relative path resolves too. */ | ||
|
|
@@ -779,11 +797,11 @@ export function verifierTools( | |
| graph.parents.find((p) => p.name === name && (p.kind === "model" || p.kind === "snapshot"))?.path ?? | ||
| graph.downstream.find((d) => d.name === name)?.path | ||
| const side = z.enum(["old", "new"]).optional().describe("old=base, new=head (default)") | ||
| const scope = NA("outside review scope (changed files, their dbt parents and direct downstream models only)") | ||
| const scope = NA("outside review scope (files the PR changes, their dbt parents and direct downstream models only)") | ||
|
|
||
| return { | ||
| read_file: { | ||
| description: "Up to 200 numbered lines of a changed file, its dbt parents or direct children.", | ||
| description: "Up to 200 numbered lines of any file the PR changes (tests and CI included), a dbt parent or direct child.", | ||
| input: z.object({ | ||
| path: z.string(), | ||
| side, | ||
|
|
@@ -1158,6 +1176,7 @@ export async function runAiVerify( | |
| finding: input.findings[i], | ||
| diff: fileByPath.get(input.findings[i].file)?.diff, | ||
| })), | ||
| ev.prFiles, | ||
| ) | ||
| return { idx, ids, message } | ||
| }) | ||
|
|
||
| Original file line number | Diff line number | Diff line change | ||||
|---|---|---|---|---|---|---|
|
|
@@ -964,7 +964,9 @@ export function redundantJoinPredicates(sql: string): string[] { | |||||
| } else if (rel) i += rel[0].length | ||||||
| const tail = /^\s+(?:as\s+)?(\w+)\s+on\b/i.exec(s.slice(i)) ?? /^\s+on\b/i.exec(s.slice(i)) | ||||||
| if (!tail) { | ||||||
| joins.push({ preds: [], start: m.index!, end: i, inner }) | ||||||
| // A join with no ON (`cross join params as p`): step over its alias so the chain continues. | ||||||
| const bare = /^\s+(?:as\s+)?(?!(?:natural|inner|left|right|full|cross|outer|join|lateral|on|using|where|group|order|qualify|having|union|limit|window|select|from)\b)\w+\b/i.exec(s.slice(i)) | ||||||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. P2: A Prompt for AI agents
Suggested change
|
||||||
| joins.push({ preds: [], start: m.index!, end: i + (bare?.[0].length ?? 0), inner }) | ||||||
| continue | ||||||
| } | ||||||
| const alias = tail[1] && tail[1].toLowerCase() !== "on" ? tail[1].toLowerCase() : rel?.[1].split(".").pop()?.toLowerCase() | ||||||
|
|
||||||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -18,6 +18,7 @@ import { | |
| type NotReviewed, | ||
| partitionChangedFiles, | ||
| hasOldSide, | ||
| isExcludedPath, | ||
| walkDiffLines, | ||
| walkDiffRows, | ||
| } from "./diff-filter" | ||
|
|
@@ -285,6 +286,8 @@ export interface OrchestrateInput { | |
| aiVerify?: boolean | ||
| /** Fixed-string search over the dbt project at head, for the verifier. */ | ||
| grepProject?: AiVerifyEvidence["grepProject"] | ||
| /** Every file the PR changes (repo-relative), including files outside the dbt project; the verifier may read them. */ | ||
| prFiles?: string[] | ||
| /** PR metadata passed to the AI reviewer for intent checking. */ | ||
| prTitle?: string | ||
| prBody?: string | ||
|
|
@@ -298,6 +301,8 @@ export interface OrchestrateInput { | |
| layout?: DbtProjectLayout | ||
| /** Names `ref()` can resolve to at head (repo resources + installed packages); undefined = unknown. */ | ||
| resourceNames?: () => Promise<Set<string> | undefined> | ||
| /** `{% docs %}` names installed packages define; undefined when that can't be known. */ | ||
| packageDocNames?: () => Promise<Set<string> | undefined> | ||
| /** What the review covered: base resolution and files left out of scope. */ | ||
| scope?: Omit<ReviewScope, "unverified" | "notReviewed"> & { notReviewed?: NotReviewed[] } | ||
| /** Drop the regex catalog's cross-engine portability findings (single known engine). */ | ||
|
|
@@ -2352,20 +2357,31 @@ export async function runReview(input: OrchestrateInput): Promise<VerdictEnvelop | |
| // dozens of yml files; serial `git show` per file adds up). | ||
| // A `{% docs name %}` block in any of the project's .md files at head (cached per name). | ||
| const docLookups = new Map<string, Promise<boolean>>() | ||
| // Docs blocks of installed packages (cached); undefined when they can't be known. | ||
| let pkgDocs: Promise<Set<string> | undefined> | undefined | ||
| const packageDocs = () => (pkgDocs ??= input.packageDocNames?.().catch(() => undefined) ?? Promise.resolve(undefined)) | ||
| const docExists = (name: string) => { | ||
| if (!docLookups.has(name)) { | ||
| const block = new RegExp(`\\{%-?\\s*docs\\s+${name}\\s*-?%\\}`) | ||
| docLookups.set( | ||
| name, | ||
| (input.grepProject?.(`docs ${name}`, "*.md") ?? Promise.resolve([])) | ||
| // An unavailable search (undefined) is no evidence the block is missing. | ||
| .then((hits) => hits === undefined || hits.some((h) => block.test(h.text))) | ||
| .then(async (hits) => hits === undefined || hits.some((h) => block.test(h.text)) || !!(await packageDocs())?.has(name)) | ||
| // A failed search is no evidence the block is missing. | ||
| .catch(() => true), | ||
| ) | ||
| } | ||
| return docLookups.get(name)! | ||
| } | ||
| // The project defines docs blocks of its own (any `{% docs x %}` in its .md files), and its | ||
| // installed packages' blocks are known: a package may define the name otherwise. | ||
| let docsAny: Promise<boolean> | undefined | ||
| const projectHasDocs = () => | ||
| (docsAny ??= (input.grepProject?.("docs", "*.md") ?? Promise.resolve([])) | ||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. SUGGESTION: the generic
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. P2: The generic Prompt for AI agents |
||
| .then(async (hits) => !!hits?.some((h) => /\{%-?\s*docs\s+\w+\s*-?%\}/.test(h.text)) && (await packageDocs()) !== undefined) | ||
| // A failed or unavailable search is no evidence: stay silent. | ||
| .catch(() => false)) | ||
| const schemaFiles = reviewable.filter((f) => f.kind === "schema_yml") | ||
| touch(schemaFiles) | ||
| const sqlByModel = new Map([...ctxByPath.values()].filter((c) => c.newSql).map((c) => [modelNameFromPath(c.file.path).toLowerCase(), c.newSql!])) | ||
|
|
@@ -2384,7 +2400,7 @@ export async function runReview(input: OrchestrateInput): Promise<VerdictEnvelop | |
| ...detectUnitTestFixtures(file, oldContent, newContent), | ||
| ...snapshotYmlFindings(file, oldContent, newContent), | ||
| ...detectTestAdequacy({ file, oldContent, newContent, projectYml, layout: input.layout }), | ||
| ...(lanes.has("dbt_patterns") && input.grepProject ? await docRefFindings(file, oldContent, newContent, docExists) : []), | ||
| ...(lanes.has("dbt_patterns") && input.grepProject ? await docRefFindings(file, oldContent, newContent, docExists, projectHasDocs) : []), | ||
| ] | ||
| } catch (err) { | ||
| // One malformed schema file degrades that file only. | ||
|
|
@@ -2571,6 +2587,10 @@ export async function runReview(input: OrchestrateInput): Promise<VerdictEnvelop | |
| touch(aiFiles) | ||
| if (input.aiContextCard && anyManifest) await attachContextCards(aiFiles, input.runner) | ||
| const context = input.aiGraphContext ? await aiGraphContext(aiFiles.map((f) => f.path), input.runner, getContent) : undefined | ||
| // Files the PR changes that the AI may list and the verifier may read; never the user's exclusions. | ||
| const prFiles = (input.prFiles ?? input.changedFiles.map((f) => f.path)).filter( | ||
| (p) => !isExcludedPath(p, input.rubric.exclusions.excludeGlobs), | ||
| ) | ||
| // The verifier reads engine SQL first (the same SQL the engine lanes saw), then compiled output. | ||
| const evidence: AiVerifyEvidence | undefined = input.aiVerify | ||
| ? { | ||
|
|
@@ -2582,6 +2602,7 @@ export async function runReview(input: OrchestrateInput): Promise<VerdictEnvelop | |
| prTitle: input.prTitle, | ||
| prBody: input.prBody, | ||
| grepProject: input.grepProject, | ||
| prFiles, | ||
| readFile: async (p, side, start, end) => numberedSlice(await getContent?.(p, side), start, end), | ||
| compiledSql: async (p, side) => { | ||
| const ctx = ctxByPath.get(p) | ||
|
|
@@ -2598,6 +2619,7 @@ export async function runReview(input: OrchestrateInput): Promise<VerdictEnvelop | |
| prTitle: input.prTitle, | ||
| prBody: input.prBody, | ||
| model: input.aiModel, | ||
| prFiles, | ||
| ...(evidence ? { evidence } : {}), | ||
| }) | ||
| const aiFindings = Array.isArray(res) ? res : res.findings | ||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -390,7 +390,11 @@ export function yamlParseFindings(file: ChangedFile, oldContent: string | undefi | |
| if (!/\.ya?ml$/i.test(file.path) || file.status === "deleted" || !newContent) return [] | ||
| const err = yamlError(newContent) | ||
| if (!err || (oldContent !== undefined && yamlError(oldContent))) return [] | ||
| const text = err.line ? newContent.split("\n")[err.line - 1] : undefined | ||
| // Some errors carry no line, or one past the edit (an unterminated quote fails where the scan | ||
| // ends): anchor on the added line at or before it, else the first added line. | ||
| const added = [...walkDiffRows(file.diff)].filter((r) => r.kind === "+") | ||
| const at = (err.line ? added.find((r) => r.line === err.line) ?? [...added].reverse().find((r) => r.line < err.line!) : undefined) ?? added[0] | ||
| const text = at?.text ?? (err.line ? newContent.split("\n")[err.line - 1] : undefined) | ||
| const f = finding( | ||
| file, | ||
| "yaml-parse-error", | ||
|
|
@@ -402,7 +406,7 @@ export function yamlParseFindings(file: ChangedFile, oldContent: string | undefi | |
| : `${err.message}${err.line ? ` (line ${err.line})` : ""}. The file parsed before this change; tools that read it now fail.`, | ||
| text, | ||
| ) | ||
| return [err.line ? { ...f, startLine: err.line, endLine: err.line } : f] | ||
| return [at ? { ...f, startLine: at.line, endLine: at.line } : f] | ||
| } | ||
|
|
||
| /** New models named like the public models a `.quickstart/quickstart.yml` lists, but missing from | ||
|
|
@@ -477,14 +481,17 @@ export function packageLockFindings(file: ChangedFile, isPackage: boolean): Find | |
|
|
||
| const DOC_REF = /\bdoc\s*\(\s*['"](\w+)['"]\s*\)/g | ||
|
|
||
| /** `doc('x')` names this change adds to a yml file that no docs block in the project defines: | ||
| * dbt fails to parse the project. Only when another `doc()` in the file resolves in the project, | ||
| * so docs kept in an installed package aren't mistaken for missing ones. */ | ||
| /** `doc('x')` names this change adds to a yml file that no docs block in the project or its | ||
| * installed packages defines: dbt fails to parse the project. Only when another `doc()` in the | ||
| * file resolves, or, when the file has no other `doc()`, the project keeps its own docs blocks | ||
| * (`projectHasDocs` also requires the packages' blocks to be known); so docs kept in an | ||
| * uninstalled package aren't mistaken for missing ones. */ | ||
| export async function docRefFindings( | ||
| file: ChangedFile, | ||
| oldContent: string | undefined, | ||
| newContent: string | undefined, | ||
| docExists: (name: string) => Promise<boolean>, | ||
| projectHasDocs?: () => Promise<boolean>, | ||
| ): Promise<Finding[]> { | ||
| if (!newContent || file.status === "deleted") return [] | ||
| const before = new Set([...(oldContent ?? "").matchAll(DOC_REF)].map((m) => m[1])) | ||
|
|
@@ -497,6 +504,7 @@ export async function docRefFindings( | |
| const others = [...new Set([...newContent.matchAll(DOC_REF)].map((m) => m[1]))].filter((n) => !missing.includes(n)).slice(0, 5) | ||
| let resolves = false | ||
| for (const n of others) if ((resolves = await docExists(n))) break | ||
| if (!others.length && projectHasDocs) resolves = await projectHasDocs() | ||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. WARNING:
cubic-dev-ai[bot] marked this conversation as resolved.
Resolved
|
||
| if (!resolves) return [] | ||
| return missing.map((name) => | ||
| finding( | ||
|
|
@@ -505,7 +513,7 @@ export async function docRefFindings( | |
| "warning", | ||
| "sql_correctness", | ||
| `\`doc('${name}')\` has no docs block`, | ||
| `No \`{% docs ${name} %}\` block exists in the project's \`.md\` files, while the other \`doc()\` references in this file resolve there. dbt fails to parse the project on a \`doc()\` it can't resolve. Add the docs block, or fix the name.`, | ||
| `No \`{% docs ${name} %}\` block exists in the project's \`.md\` files or its installed packages, while ${others.length ? "the other \`doc()\` references in this file resolve" : "the project defines its own docs blocks"}. dbt fails to parse the project on a \`doc()\` it can't resolve. Add the docs block, or fix the name.`, | ||
| lines.get(name), | ||
| ), | ||
| ) | ||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -75,12 +75,15 @@ export function blockingCategories(rubric: Rubric): Set<string> { | |
| * be dropped, or null to keep it. Centralizing this here (vs prompt text) is | ||
| * what keeps the false-positive rate down deterministically. | ||
| */ | ||
| /** Findings that break `dbt parse` for the whole project, so a dev/sandbox/scratch folder doesn't excuse them. */ | ||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. SUGGESTION: the The |
||
| const PROJECT_BREAKING_RULES = new Set(["dbt-project-rules:yaml-parse-error", "dbt-project-rules:doc-block-missing"]) | ||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. P3: Move Prompt for AI agents |
||
|
|
||
| export function exclusionReason(finding: Finding, rubric: Rubric): string | null { | ||
| const ex = rubric.exclusions | ||
| const isStaging = /(^|\/)stg_|(^|\/)staging\//.test(finding.file) || finding.model?.startsWith("stg_") | ||
| const isDev = /(^|\/)(dev|sandbox|scratch)\//.test(finding.file) | ||
|
|
||
| if (ex.skipNonProdModels && isDev) return "non-prod model (dev/sandbox/scratch)" | ||
| if (ex.skipNonProdModels && isDev && !PROJECT_BREAKING_RULES.has(finding.rule ?? "")) return "non-prod model (dev/sandbox/scratch)" | ||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. WARNING: This exemption only keeps |
||
|
|
||
| if ( | ||
| ex.allowSelectStarInStaging && | ||
|
|
||
| Original file line number | Diff line number | Diff line change | ||||
|---|---|---|---|---|---|---|
|
|
@@ -429,6 +429,7 @@ export async function reviewPullRequest(opts: ReviewPullRequestOptions): Promise | |||||
|
|
||||||
| return runReview({ | ||||||
| resourceNames: () => projectResourceNames(opts.cwd, opts.head, projectRootRel, dbtRoot), | ||||||
| packageDocNames: () => packageDocNames(opts.cwd, opts.head, projectRootRel, dbtRoot), | ||||||
| changedFiles, | ||||||
| config, | ||||||
| rubric, | ||||||
|
|
@@ -460,6 +461,7 @@ export async function reviewPullRequest(opts: ReviewPullRequestOptions): Promise | |||||
| const specs = projectGrepSpecs(projectRootRel, dirs, glob) | ||||||
| return specs ? grepFixedAt(gitRoot, opts.head, pattern, specs) : Promise.resolve(undefined) | ||||||
| }, | ||||||
| prFiles: collected.map((f) => f.path), | ||||||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. P2: Prompt for AI agents
Suggested change
|
||||||
| prTitle: opts.prTitle, | ||||||
| prBody: opts.prBody, | ||||||
| explainTier: opts.explainTier, | ||||||
|
|
@@ -599,6 +601,41 @@ export async function projectAdapters( | |||||
| return [...seen].sort() | ||||||
| } | ||||||
|
|
||||||
| /** | ||||||
| * `{% docs name %}` blocks defined in the `.md` files of installed packages (`dbt_packages/`). Empty when the | ||||||
| * project declares no packages; undefined when that can't be known (no project root, git can't | ||||||
| * list the tree, or packages are declared but not installed). | ||||||
| */ | ||||||
| export async function packageDocNames( | ||||||
| cwd: string, | ||||||
| head: string | undefined, | ||||||
| projectRootRel: string | undefined, | ||||||
| dbtRoot: string, | ||||||
| ): Promise<Set<string> | undefined> { | ||||||
| if (projectRootRel === undefined) return undefined | ||||||
| const files = await listFilesAt(cwd, head) | ||||||
| if (!files) return undefined | ||||||
| const prefix = projectRootRel ? projectRootRel + "/" : "" | ||||||
| const names = new Set<string>() | ||||||
| if (!files.includes(prefix + "packages.yml") && !files.includes(prefix + "dependencies.yml")) return names | ||||||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. SUGGESTION: a working-tree review treats an uncommitted
|
||||||
| const walk = async (dir: string, depth: number): Promise<boolean> => { | ||||||
| let entries | ||||||
| try { | ||||||
| entries = await readdir(dir, { withFileTypes: true }) | ||||||
| } catch { | ||||||
| return false | ||||||
| } | ||||||
| for (const e of entries) { | ||||||
| const p = path.join(dir, e.name) | ||||||
| if (e.isDirectory() && depth < 12 && e.name !== ".git") await walk(p, depth + 1) | ||||||
| else if (e.isFile() && e.name.endsWith(".md")) | ||||||
| for (const m of (await readFile(p, "utf8").catch(() => "")).matchAll(/\{%-?\s*docs\s+(\w+)\s*-?%\}/g)) names.add(m[1]) | ||||||
| } | ||||||
| return true | ||||||
| } | ||||||
| return (await walk(path.join(dbtRoot, "dbt_packages"), 0)) ? names : undefined | ||||||
| } | ||||||
|
|
||||||
| /** | ||||||
| * Names `ref()` can resolve to: model/seed/snapshot files in the repo at head, snapshots | ||||||
| * defined in YAML, and | ||||||
|
|
||||||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
P2: The verifier cannot discover changed files after the first 40 paths, so a test added at path 41+ can be missed by the test-gap check; add a bounded file-list tool or otherwise expose all paths to the verifier.
Prompt for AI agents