Repository navigation
fix(review): review the PR's own changes: real base, merge-base diff, project scope, renames, model-paths, no raw Jinja #1719
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: main
Are you sure you want to change the base?
Changes from all commits
0c9314f
0f68b0c
914d205
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -2,6 +2,7 @@ import { promises as fs } from "node:fs" | |
| import path from "node:path" | ||
| import YAML from "yaml" | ||
| import { safeReadInside } from "./git" | ||
| import { DEFAULT_DBT_DIRS, type DbtProjectLayout } from "./diff-filter" | ||
|
|
||
| /** | ||
| * Resolve dbt's COMPILED SQL for static analysis. | ||
|
|
@@ -17,8 +18,8 @@ import { safeReadInside } from "./git" | |
| * the base ref compiled into a sibling target dir) | ||
| * | ||
| * When no compiled artifact exists (no `dbt compile` ran, or a raw single-file | ||
| * diff), the caller falls back to raw Jinja and the engine result is treated as | ||
| * approximate/undecidable — never fabricated. A proper offline renderer | ||
| * diff), raw Jinja is never sent to the engine: a templated model skips the | ||
| * engine lanes and is recorded in `scope.unverified`. A proper offline renderer | ||
| * (minijinja + dbt builtin stubs, à la dbt-Fusion) is the documented future | ||
| * fallback; it is intentionally NOT a regex strip. | ||
| */ | ||
|
|
@@ -36,6 +37,39 @@ export async function dbtProjectName(cwd: string): Promise<string | undefined> { | |
| return undefined | ||
| } | ||
|
|
||
| const LAYOUT_KEYS: Record<keyof DbtProjectLayout["dirs"], string> = { | ||
| model: "model-paths", | ||
| snapshot: "snapshot-paths", | ||
| seed: "seed-paths", | ||
| macro: "macro-paths", | ||
| test: "test-paths", | ||
| analysis: "analysis-paths", | ||
| } | ||
|
|
||
| /** | ||
| * Read the project's directory layout (`model-paths`, `snapshot-paths`, …) | ||
| * from dbt_project.yml, falling back to dbt's defaults per key. `root` is the | ||
| * project directory relative to the repository root. Undefined when there is | ||
| * no readable dbt_project.yml in `projectDir`. | ||
| */ | ||
| export async function dbtProjectLayout(projectDir: string, root: string): Promise<DbtProjectLayout | undefined> { | ||
| for (const f of ["dbt_project.yml", "dbt_project.yaml"]) { | ||
| let doc: any | ||
| try { | ||
| doc = YAML.parse(await fs.readFile(path.join(projectDir, f), "utf8")) | ||
| } catch { | ||
| continue | ||
| } | ||
| const dirs = { ...DEFAULT_DBT_DIRS } | ||
| for (const [role, key] of Object.entries(LAYOUT_KEYS) as Array<[keyof typeof dirs, string]>) { | ||
| const v = doc?.[key] | ||
| if (Array.isArray(v) && v.length && v.every((x) => typeof x === "string")) dirs[role] = v | ||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. P1: This records Jinja path expressions without rendering them. Projects using Prompt for AI agents
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Follow-up: low priority, tracked. Templated |
||
| } | ||
| return { root: root.replace(/\\/g, "/"), dirs } | ||
| } | ||
| return undefined | ||
| } | ||
|
|
||
| export interface CompiledResolverOptions { | ||
| /** dbt project root (the dir containing `dbt_project.yml`). */ | ||
| cwd: string | ||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -55,23 +55,88 @@ export type DbtFileKind = | |
| | "project_config" // dbt_project.yml / profiles.yml / packages.yml | ||
| | "other" | ||
|
|
||
| /** Classify a changed file by its role in a dbt project. */ | ||
| export function classifyDbtFile(path: string): DbtFileKind { | ||
| const p = path.replace(/\\/g, "/").toLowerCase() | ||
| type DbtRole = "model" | "snapshot" | "seed" | "macro" | "test" | "analysis" | ||
|
|
||
| /** | ||
| * Where a dbt project keeps each kind of file, from `dbt_project.yml` | ||
| * (`model-paths`, `snapshot-paths`, …). `root` is the project directory | ||
| * relative to the repository root ("" when the project is at the root). | ||
| */ | ||
| export interface DbtProjectLayout { | ||
| root: string | ||
| dirs: Record<DbtRole, string[]> | ||
| } | ||
|
|
||
| export const DEFAULT_DBT_DIRS: Record<DbtRole, string[]> = { | ||
| model: ["models"], | ||
| snapshot: ["snapshots"], | ||
| seed: ["seeds"], | ||
| macro: ["macros"], | ||
| test: ["tests"], | ||
| analysis: ["analyses"], | ||
| } | ||
|
|
||
| const norm = (p: string) => p.replace(/\\/g, "/") | ||
|
|
||
| /** `p` relative to `root`, or undefined when it lies outside. Both POSIX, repo-relative. */ | ||
| export function relativeToRoot(p: string, root: string): string | undefined { | ||
| const r = norm(root).replace(/^\.\/?/, "").replace(/\/+$/, "") | ||
| const f = norm(p) | ||
| if (!r) return f | ||
| return f.startsWith(r + "/") ? f.slice(r.length + 1) : undefined | ||
| } | ||
|
|
||
| /** The role of a project-relative path from the configured directories; the | ||
| * longest matching directory wins so `models/tests/x.sql` is a model when | ||
| * `models` is a model path. */ | ||
| function roleFromLayout(rel: string, dirs: Record<DbtRole, string[]>): DbtRole | undefined { | ||
| const p = rel.toLowerCase() | ||
| let best: { role: DbtRole; len: number } | undefined | ||
| for (const [role, list] of Object.entries(dirs) as Array<[DbtRole, string[]]>) { | ||
| for (const d of list) { | ||
| const dir = norm(d).toLowerCase().replace(/^\.\/?/, "").replace(/\/+$/, "") | ||
| // `model-paths: ["."]` normalises to "": it matches every path at length 0, | ||
| // so a more specific directory still wins. | ||
| const matches = dir === "" || p.startsWith(dir + "/") | ||
| if (matches && (!best || dir.length > best.len)) best = { role, len: dir.length } | ||
| } | ||
| } | ||
| return best?.role | ||
| } | ||
|
|
||
| /** Without a layout, the OUTERMOST conventional directory decides the role, | ||
| * so `models/tests/x.sql` is a model and `tests/models/x.sql` a test. */ | ||
| function roleFromConvention(p: string): DbtRole | undefined { | ||
| const segments = p.toLowerCase().split("/").slice(0, -1) | ||
| for (const seg of segments) { | ||
| for (const [role, list] of Object.entries(DEFAULT_DBT_DIRS) as Array<[DbtRole, string[]]>) { | ||
| if (list.includes(seg)) return role | ||
| } | ||
| } | ||
| return undefined | ||
| } | ||
|
|
||
| /** Classify a changed file by its role in a dbt project. With a layout, the | ||
| * project's own `*-paths` decide (custom `model-paths` are honoured); | ||
| * without one, conventional directory names do. */ | ||
| export function classifyDbtFile(path: string, layout?: DbtProjectLayout): DbtFileKind { | ||
| const p = norm(path).toLowerCase() | ||
| const isYaml = p.endsWith(".yml") || p.endsWith(".yaml") | ||
| if (/(^|\/)(dbt_project|profiles|packages|dependencies)\.ya?ml$/.test(p)) return "project_config" | ||
| if (/(^|\/)macros\//.test(p)) return "macro" | ||
| const rel = layout ? relativeToRoot(norm(path), layout.root) : undefined | ||
| const role = rel !== undefined && layout ? roleFromLayout(rel, layout.dirs) : roleFromConvention(p) | ||
|
cubic-dev-ai[bot] marked this conversation as resolved.
Resolved
|
||
| if (role === "macro") return "macro" | ||
| // Snapshots split by extension: `.sql` files are snapshot definitions | ||
| // (tier-forcing, catalog-scoped), YAML property files under `snapshots/` | ||
| // are schema files (declared tests + column metadata) and should route to | ||
| // `schema_yml` so the test-removal detector runs on them. | ||
| if (/(^|\/)snapshots\//.test(p) && p.endsWith(".sql")) return "snapshot" | ||
| if (/(^|\/)seeds\//.test(p) && p.endsWith(".csv")) return "seed" | ||
| if (/(^|\/)tests\//.test(p) && p.endsWith(".sql")) return "test" | ||
| if (/(^|\/)analyses\//.test(p)) return "analysis" | ||
| if (/(^|\/)models\//.test(p) && p.endsWith(".py")) return "python_model" | ||
| if (/(^|\/)models\//.test(p) && p.endsWith(".sql")) return "model_sql" | ||
| if (isYaml && /(^|\/)(models|snapshots|seeds|tests)\//.test(p)) return "schema_yml" | ||
| if (role === "snapshot" && p.endsWith(".sql")) return "snapshot" | ||
| if (role === "seed" && p.endsWith(".csv")) return "seed" | ||
| if (role === "test" && p.endsWith(".sql")) return "test" | ||
| if (role === "analysis") return "analysis" | ||
| if (role === "model" && p.endsWith(".py")) return "python_model" | ||
| if (role === "model" && p.endsWith(".sql")) return "model_sql" | ||
| if (isYaml && (role === "model" || role === "snapshot" || role === "seed" || role === "test")) return "schema_yml" | ||
| if (isYaml && /(^|\/)(_?schema|_?models|_?sources|sources|properties)\.ya?ml$/.test(p)) return "schema_yml" | ||
| return "other" | ||
| } | ||
|
|
@@ -84,6 +149,21 @@ export interface ChangedFile { | |
| diff?: string | ||
| /** Previous path for renames. */ | ||
| oldPath?: string | ||
| /** Rename similarity (0–100) from git; 100 means the content is unchanged. */ | ||
| similarity?: number | ||
| /** dbt role, set by filterChangedFiles from the project layout. */ | ||
| kind?: DbtFileKind | ||
| } | ||
|
|
||
| /** The file's dbt role: the layout-aware kind when filterChangedFiles set it. */ | ||
| export function fileKind(file: ChangedFile): DbtFileKind { | ||
| return file.kind ?? classifyDbtFile(file.path) | ||
| } | ||
|
|
||
| /** True when a file has a previous version to compare against: a modification, | ||
| * or a rename (whose old side lives at `oldPath`). */ | ||
| export function hasOldSide(file: ChangedFile): boolean { | ||
| return file.status === "modified" || (file.status === "renamed" && !!file.oldPath) | ||
| } | ||
|
|
||
| // Compile a glob (`**`, `*`, `?`) to an anchored RegExp with path semantics: | ||
|
|
@@ -119,14 +199,100 @@ function globToRegExp(glob: string): RegExp { | |
| export function filterChangedFiles( | ||
| files: ChangedFile[], | ||
| extraExcludeGlobs: string[] = [], | ||
| layout?: DbtProjectLayout, | ||
| ): Array<ChangedFile & { kind: DbtFileKind }> { | ||
| const excluders = extraExcludeGlobs.map(globToRegExp) | ||
| // Normalize to forward slashes so `/`-based glob semantics hold on Windows paths. | ||
| const norm = (p: string) => p.replace(/\\/g, "/") | ||
| return files | ||
| .filter((f) => shouldReview(f.path)) | ||
| .filter((f) => !excluders.some((re) => re.test(norm(f.path)))) | ||
| .map((f) => ({ ...f, kind: classifyDbtFile(f.path) })) | ||
| .map((f) => ({ ...f, kind: classifyDbtFile(f.path, layout) })) | ||
| } | ||
|
|
||
| /** The first folder under the file's role directory (`staging` for | ||
| * `models/staging/x.sql`), or "" for a file directly in it. Folder-level | ||
| * config in dbt_project.yml and layer conventions key off this folder. */ | ||
| function layerOf(path: string, root: string | undefined, layout?: DbtProjectLayout): string { | ||
| const rel = (relativeToRoot(path, root ?? "") ?? norm(path)).toLowerCase() | ||
| const dirs = Object.values(layout?.dirs ?? DEFAULT_DBT_DIRS) | ||
| .flat() | ||
| .map((d) => norm(d).toLowerCase().replace(/^\.\/?/, "").replace(/\/+$/, "")) | ||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. SUGGESTION: This duplicates Extracting one |
||
| .filter((d) => d && rel.startsWith(d + "/")) | ||
| .sort((a, b) => b.length - a.length) | ||
| const rest = dirs.length ? rel.slice(dirs[0].length + 1) : rel | ||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. SUGGESTION: The layer comparison only looks at the first folder below the role directory, so a move deeper inside one layer is still dropped as a pure move even though dbt path config is nested at any depth.
Since model moves are already always kept, consider dropping only roles whose folder path cannot carry config (macros/analyses), or comparing the full parent directory and documenting the limitation. |
||
| const segments = rest.split("/") | ||
| return segments.length > 1 ? segments[0] : "" | ||
| } | ||
|
|
||
| export interface ScopeResult { | ||
| /** Files inside the dbt project, with unchanged moves removed. */ | ||
| inScope: ChangedFile[] | ||
| /** Changed files outside the dbt project root (counted, not reviewed). */ | ||
| outOfScope: number | ||
| /** Non-model renames with identical content that keep the name, role and layer (a pure move). */ | ||
| movedUnchanged: number | ||
| } | ||
|
|
||
| const isModel = (p: string, layout?: DbtProjectLayout) => { | ||
| const kind = classifyDbtFile(p, layout) | ||
| return kind === "model_sql" || kind === "python_model" | ||
| } | ||
|
|
||
| /** | ||
| * Restrict the PR's changed files to the dbt project being reviewed. | ||
| * | ||
| * - Files outside `root` belong to another project or to non-dbt code: they | ||
| * are counted, never reviewed (a monorepo with several dbt projects must not | ||
| * review its siblings). | ||
| * - Every model move stays in scope, even a same-name 100% rename: a model's | ||
| * folder can carry config in `dbt_project.yml` (materialization, schema, | ||
| * tags) and match risk paths, so moving it can change how it builds. | ||
| * - A non-model rename at 100% similarity that keeps the file name, its dbt | ||
| * role, and its layer (the first folder under the role directory) is a pure | ||
| * move and is dropped: re-reviewing it would only repeat pre-existing | ||
| * findings. A move across roles or layers stays in scope. | ||
| * - A rename across the project boundary is a deletion from one project and an | ||
| * addition to the other: only the side inside `root` is kept, rewritten as | ||
| * `deleted` (old path) or `added` (new path). | ||
| */ | ||
| export function scopeToProject( | ||
| files: ChangedFile[], | ||
| root: string | undefined, | ||
| layout?: DbtProjectLayout, | ||
| ): ScopeResult { | ||
| let outOfScope = 0 | ||
| let movedUnchanged = 0 | ||
| const inScope: ChangedFile[] = [] | ||
| const basename = (p: string) => norm(p).split("/").pop() | ||
| const isInside = (p: string) => root === undefined || relativeToRoot(p, root) !== undefined | ||
| for (const f of files) { | ||
| const inside = isInside(f.path) | ||
| // A rename moving a file into or out of the project still concerns it. | ||
| const oldInside = !!f.oldPath && isInside(f.oldPath) | ||
| if (!inside && !oldInside) { | ||
| outOfScope++ | ||
| continue | ||
| } | ||
| if (f.oldPath && inside !== oldInside) { | ||
| const { oldPath, similarity: _similarity, ...rest } = f | ||
| inScope.push(inside ? { ...rest, status: "added" } : { ...rest, path: oldPath, status: "deleted" }) | ||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. P2: Cross-project rename records retain Prompt for AI agents |
||
| continue | ||
| } | ||
| if ( | ||
| f.status === "renamed" && | ||
| f.similarity === 100 && | ||
| f.oldPath && | ||
| !isModel(f.path, layout) && | ||
| basename(f.oldPath) === basename(f.path) && | ||
| classifyDbtFile(f.oldPath, layout) === classifyDbtFile(f.path, layout) && | ||
| layerOf(f.oldPath, root, layout) === layerOf(f.path, root, layout) | ||
| ) { | ||
| movedUnchanged++ | ||
| continue | ||
| } | ||
| inScope.push(f) | ||
|
coderabbitai[bot] marked this conversation as resolved.
Resolved
|
||
| } | ||
| return { inScope, outOfScope, movedUnchanged } | ||
| } | ||
|
|
||
| /** Heuristic: does a yml diff declare dbt `sources:`? Tolerates unified-diff | ||
|
|
||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
P2: This layout reader drops the legacy
source-pathsanddata-pathsaliases that the repository still supports. Older projects using those keys fall back to the default directories, causing custom model files to miss model review; honor the aliases when selecting each layout list.Prompt for AI agents
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Follow-up: low priority, tracked. These are dbt <1.0 aliases; the plan is
model-paths ?? source-pathsandseed-paths ?? data-paths.