Repository navigation
feat(review): close remaining review gaps: action dbt_prepare, data-test adequacy, parse-only column changes #1726
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: feat/review-incremental-semantics
Are you sure you want to change the base?
Changes from all commits
de458ec
e1dfd69
bf31606
e77d121
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -31,6 +31,17 @@ inputs: | |
| description: "Post the verdict to the PR (summary comment + inline review)." | ||
| required: false | ||
| default: "true" | ||
| dbt_prepare: | ||
| description: "Build the dbt artifacts the full review uses before reviewing. `none` (default): you build them in earlier steps. `compile`: `dbt compile` on the head, and on the PR's merge-base into `target-base/` (equivalence and lineage proofs). `full`: also `dbt docs generate` for `catalog.json` (complete column types). `compile` usually works offline; `full` needs warehouse credentials in the profile." | ||
| required: false | ||
| default: "none" | ||
| dbt_command: | ||
| description: "How to invoke dbt for `dbt_prepare`, e.g. `dbt`, `uv run dbt`, `poetry run dbt`." | ||
| required: false | ||
| default: "dbt" | ||
| dbt_profiles_dir: | ||
| description: "`--profiles-dir` for `dbt_prepare`, relative to `project_dir`. Empty = dbt's default lookup." | ||
| required: false | ||
| signing_key: | ||
| description: "Optional HMAC key to sign the verdict envelope (ALTIMATE_REVIEW_SIGNING_KEY)." | ||
| required: false | ||
|
|
@@ -167,6 +178,57 @@ runs: | |
| echo "Advisory lane: no model/credentials provided — running deterministic-only." | ||
| fi | ||
|
|
||
| # Optional: build the head and base artifacts the full-tier review needs. | ||
| # Inputs are passed as ENV, never interpolated into the script. | ||
| - name: Prepare dbt artifacts | ||
| if: inputs.dbt_prepare != 'none' | ||
| shell: bash | ||
| env: | ||
| IN_PREPARE: ${{ inputs.dbt_prepare }} | ||
| IN_DBT: ${{ inputs.dbt_command }} | ||
| IN_PROFILES: ${{ inputs.dbt_profiles_dir }} | ||
| IN_PROJECT_DIR: ${{ inputs.project_dir }} | ||
| IN_BASE: ${{ inputs.base }} | ||
| EVENT_BASE: ${{ github.event.pull_request.base.ref }} | ||
| run: | | ||
| set -euo pipefail | ||
| case "$IN_PREPARE" in compile|full) ;; *) echo "::error::dbt_prepare must be none, compile or full (got '$IN_PREPARE')"; exit 1 ;; esac | ||
| read -ra DBT <<< "$IN_DBT" | ||
| PROJECT="$(cd "${IN_PROJECT_DIR:-.}" && pwd)" | ||
| REL="$(git -C "$PROJECT" rev-parse --show-prefix)" | ||
| PROFILES=() | ||
| [[ -n "${IN_PROFILES:-}" ]] && PROFILES=(--profiles-dir "$(cd "$PROJECT" && cd "$IN_PROFILES" && pwd)") | ||
| build() { # $1 = project dir, $2 = target path | ||
| ( cd "$1" | ||
| [[ -f packages.yml || -f dependencies.yml ]] && "${DBT[@]}" deps ${PROFILES[@]+"${PROFILES[@]}"} | ||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. SUGGESTION:
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Not changing: |
||
| "${DBT[@]}" compile --target-path "$2" ${PROFILES[@]+"${PROFILES[@]}"} ) | ||
| } | ||
| echo "Compiling the head…" | ||
| build "$PROJECT" "$PROJECT/target" | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. [MINOR] build "$PROJECT" "$PROJECT/target"
|
||
| if [[ "$IN_PREPARE" == "full" ]]; then | ||
| echo "Generating catalog.json…" | ||
| ( cd "$PROJECT" && "${DBT[@]}" docs generate --target-path "$PROJECT/target" ${PROFILES[@]+"${PROFILES[@]}"} ) || | ||
| echo "::warning::dbt docs generate failed; continuing without catalog.json." | ||
| fi | ||
| # The base side: the PR's merge-base, compiled into target-base/. | ||
| BASE="${IN_BASE:-$EVENT_BASE}" | ||
| BASE="${BASE#origin/}" | ||
| if [[ -z "$BASE" ]]; then echo "::warning::No base branch known; skipping the base compile."; exit 0; fi | ||
| git fetch --no-tags origin "+refs/heads/$BASE:refs/remotes/origin/$BASE" >/dev/null 2>&1 || true | ||
| MB="$(git merge-base "origin/$BASE" HEAD 2>/dev/null || git merge-base "$BASE" HEAD 2>/dev/null || true)" | ||
| if [[ -z "$MB" && "$(git rev-parse --is-shallow-repository)" == "true" ]]; then | ||
| git fetch --no-tags --deepen=1000 origin "+refs/heads/$BASE:refs/remotes/origin/$BASE" "$(git rev-parse HEAD)" >/dev/null 2>&1 || true | ||
| MB="$(git merge-base "origin/$BASE" HEAD 2>/dev/null || true)" | ||
| fi | ||
| if [[ -z "$MB" ]]; then echo "::warning::Could not find the merge-base with $BASE; skipping the base compile."; exit 0; fi | ||
| WT="$RUNNER_TEMP/altimate-review-base" | ||
| trap 'git worktree remove --force "$WT" >/dev/null 2>&1 || true' EXIT | ||
| git worktree add --detach "$WT" "$MB" >/dev/null || { echo "::warning::Could not check out the base; skipping the base compile."; exit 0; } | ||
| # A stale target-base/ from an earlier run must not stand in for a failed compile. | ||
| rm -rf "$PROJECT/target-base" | ||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. WARNING: The stale When |
||
| echo "Compiling the base ($BASE @ ${MB:0:12})…" | ||
| build "$WT/$REL" "$PROJECT/target-base" || echo "::warning::The base compile failed; equivalence will be marked unverified." | ||
|
cubic-dev-ai[bot] marked this conversation as resolved.
Resolved
|
||
|
|
||
| - name: Run dbt PR review | ||
| shell: bash | ||
| # Inputs are passed as ENV (data), never interpolated into the command — | ||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -225,15 +225,70 @@ const detectNotIn: Detector = (c) => { | |
| }) | ||
| } | ||
|
|
||
| /** Head-side line numbers of the added lines; undefined when the diff has no hunk headers. */ | ||
| function addedLineNumbers(diff: string | undefined): Set<number> | undefined { | ||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. SUGGESTION: This hunk walker duplicates the exported
|
||
| if (!diff || !/^@@ /m.test(diff)) return undefined | ||
| const out = new Set<number>() | ||
| let right = 0 | ||
| let inHunk = false | ||
| for (const line of diff.split("\n")) { | ||
| const h = /^@@ -\d+(?:,\d+)? \+(\d+)(?:,\d+)? @@/.exec(line) | ||
| if (h) { | ||
| right = Number(h[1]) | ||
| inHunk = true | ||
| continue | ||
| } | ||
| if (!inHunk || line.startsWith("\\")) continue | ||
| if (line.startsWith("+")) out.add(right++) | ||
| else if (line.startsWith(" ") || line === "") right++ | ||
| else if (!line.startsWith("-")) inHunk = false | ||
| } | ||
| return out | ||
| } | ||
|
|
||
| /** | ||
| * `row_number() over (…)` windows in the model, read from the whole file so a | ||
| * window split across lines is seen whole. A window counts as part of the | ||
| * change when one of the PR's added lines falls inside its line span (by text | ||
| * only when the diff carries no hunk headers). | ||
| */ | ||
| function dedupWindows(c: Ctx): Array<{ spec: string; text: string }> { | ||
| // Comments blanked with newlines kept, so offsets map to head line numbers. | ||
| const sql = c.newSql.replace(/\/\*[\s\S]*?\*\/|--[^\n]*|\{#[\s\S]*?#\}/g, (m) => m.replace(/[^\n]/g, " ")) | ||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. SUGGESTION: This comment blanking is not quote-aware; A |
||
| const lineAt = (offset: number) => sql.slice(0, offset).split("\n").length | ||
| const addedLines = addedLineNumbers(c.file.diff) | ||
| const added = c.added.map((l) => stripComments(l).trim()).filter((l) => l.length >= 3) | ||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. P2: The Prompt for AI agents
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Follow-up: low priority, tracked. When the diff has hunk headers (the PR API always sends them), matching now uses line numbers and no longer depends on the |
||
| const out: Array<{ spec: string; text: string }> = [] | ||
| for (const m of sql.matchAll(/row_number\s*\(\s*\)\s*over\s*\(/gi)) { | ||
| const open = m.index! + m[0].length - 1 | ||
| let depth = 0 | ||
| let close = -1 | ||
| for (let i = open; i < sql.length; i++) { | ||
| if (sql[i] === "(") depth++ | ||
| else if (sql[i] === ")" && --depth === 0) { | ||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. P2: The parenthesis scan counts parentheses inside SQL string literals. A value like Prompt for AI agents
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Follow-up: low priority, tracked. We'll run There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. SUGGESTION: Paren scan runs on comment-stripped but not literal-stripped SQL
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Follow-up: low priority, tracked. Same as r4626962367. |
||
| close = i | ||
| break | ||
| } | ||
| } | ||
| if (close < 0) continue | ||
| const text = sql.slice(m.index!, close + 1) | ||
| const touched = addedLines | ||
| ? [...addedLines].some((n) => n >= lineAt(m.index!) && n <= lineAt(close)) | ||
| : added.some((l) => text.includes(l) || l.includes(text.replace(/\s+/g, " "))) | ||
| if (!touched) continue | ||
| out.push({ spec: sql.slice(open + 1, close).replace(/\s+/g, " ").trim(), text: text.replace(/\s+/g, " ") }) | ||
| } | ||
| return out | ||
| } | ||
|
|
||
| // 9. Dedup via ROW_NUMBER()/QUALIFY whose ORDER BY has no unique tiebreaker. | ||
| const detectDedupTie: Detector = (c) => { | ||
| const hit = c.added.find((l) => /row_number\s*\(\s*\)\s*over\s*\(/i.test(stripComments(l))) | ||
| if (!hit) return null | ||
| // Only flag dedup with NO ORDER BY — that is unambiguously non-deterministic. | ||
| // A present ORDER BY (even single-column) is the developer's deterministic | ||
| // choice and the dbt-recommended pattern; warning "it might not be unique" | ||
| // on every dedup is noise (false positives on the common, correct case). | ||
| if (/order\s+by/i.test(stripComments(hit))) return null | ||
| const hit = dedupWindows(c).find((w) => !/order\s+by/i.test(w.spec)) | ||
| if (!hit) return null | ||
| return makeFinding({ | ||
| severity: "warning", | ||
| category: "dedup", | ||
|
|
@@ -244,7 +299,7 @@ const detectDedupTie: Detector = (c) => { | |
| file: c.file.path, | ||
| model: c.model, | ||
| confidence: "high", | ||
| evidence: { tool: "dbt-patterns", result: { rule: "dedup_no_tiebreaker", line: hit.trim() } }, | ||
| evidence: { tool: "dbt-patterns", result: { rule: "dedup_no_tiebreaker", line: hit.text } }, | ||
| ruleKey: "dedup:no-tiebreaker", | ||
| }) | ||
| } | ||
|
|
@@ -257,11 +312,8 @@ const detectDedupTie: Detector = (c) => { | |
| const LOAD_TS_RE = | ||
| /^(_?(ingest(ion|ed)?|load(ed)?|etl|extract(ed)?|batch)(_at|_ts|_time|_timestamp|_date|_dt)|_?(ingestion|load|etl|batch)_timestamp|_loaded_at)$/i | ||
| const detectDedupLoadTimestamp: Detector = (c) => { | ||
| for (const line of c.added) { | ||
| const s = stripComments(line) | ||
| const m = /row_number\s*\(\s*\)\s*over\s*\(([^)]*)\)/i.exec(s) | ||
| if (!m) continue | ||
| const order = /order\s+by\s+(.+)$/i.exec(m[1])?.[1] | ||
| for (const w of dedupWindows(c)) { | ||
| const order = /order\s+by\s+(.+)$/i.exec(w.spec)?.[1] | ||
| if (!order) continue | ||
| const keys = order.split(",").map((k) => k.trim().replace(/\s+(asc|desc)\b.*$/i, "").replace(/\s+nulls\s+(first|last)$/i, "")) | ||
| if (keys.length !== 1) continue | ||
|
|
@@ -277,7 +329,7 @@ const detectDedupLoadTimestamp: Detector = (c) => { | |
| file: c.file.path, | ||
| model: c.model, | ||
| confidence: "high", | ||
| evidence: { tool: "dbt-patterns", result: { rule: "dedup_load_timestamp_only", line: line.trim() } }, | ||
| evidence: { tool: "dbt-patterns", result: { rule: "dedup_load_timestamp_only", line: w.text } }, | ||
| ruleKey: "dedup:load-timestamp-only", | ||
| }) | ||
| } | ||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -26,6 +26,7 @@ import { detectModelPatterns, detectSchemaYmlPatterns, splitDiff } from "./dbt-p | |
| import { listMirrorsModel, type ConsumerColumnList } from "./consumer-lists" | ||
| import { detectUnitTestFixtures } from "./unit-tests" | ||
| import { selectListNames } from "./select-list" | ||
| import { detectTestAdequacy, detectThresholdLoosening } from "./test-adequacy" | ||
| import { incrementalFindings, snapshotFindings, snapshotYmlFindings } from "./incremental-semantics" | ||
| import { type AiLaneStatus, type AiReviewInput, type AiReviewResult, safeReason } from "./ai-review" | ||
|
|
||
|
|
@@ -661,19 +662,44 @@ async function columnDelta( | |
| dialect: string, | ||
| ): Promise<{ added: string[]; removed: string[]; current: Set<string> }> { | ||
| const none = { added: [], removed: [], current: new Set<string>() } | ||
| const { file, engineNewSql, engineOldSql } = ctx | ||
| if (!engineNewSql || file.status === "deleted") return none | ||
| const current = await outputColumns(runner, engineNewSql, dialect) | ||
| if (!current.size) return none | ||
| if (!hasOldSide(file)) return { added: [...current], removed: [], current } | ||
| if (!engineOldSql) return none | ||
| const before = await outputColumns(runner, engineOldSql, dialect) | ||
| if (!before.size) return none | ||
| return { | ||
| added: [...current].filter((c) => !before.has(c)), | ||
| removed: [...before].filter((c) => !current.has(c)), | ||
| current, | ||
| const { file, engineNewSql, engineOldSql, newSql, oldSql } = ctx | ||
| if (file.status === "deleted") return none | ||
| const withOld = hasOldSide(file) | ||
| const delta = (current: Set<string>, before?: Set<string>) => | ||
| !current.size || (withOld && !before?.size) | ||
| ? none | ||
| : { | ||
| added: [...current].filter((c) => !before?.has(c)), | ||
| removed: before ? [...before].filter((c) => !current.has(c)) : [], | ||
| current, | ||
| } | ||
| // Compiled SQL on both sides: column lineage plus the final SELECT's names. | ||
| if (engineNewSql && (!withOld || engineOldSql)) { | ||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. P2: When only the base compile is missing, Prompt for AI agents
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Fixed in e77d121. A |
||
| const current = await outputColumns(runner, engineNewSql, dialect) | ||
| const before = withOld ? await outputColumns(runner, engineOldSql!, dialect) : undefined | ||
| return delta(current, before) | ||
| } | ||
| // No compiled SQL (CI that only runs `dbt parse`): the raw model's own SELECT | ||
| // list, when it names its columns. Columns produced inside a macro stay unknown. | ||
| if (!newSql) return none | ||
| const current = rawSelectList(newSql) | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. [MINOR] Parse-only select list treated as complete when items are dropped const current = rawSelectList(newSql)
Fix: return unknown (empty set) when any non-empty select item produced no name. Related: stripping |
||
| const before = withOld && oldSql ? rawSelectList(oldSql) : undefined | ||
| return delta(current, before) | ||
| } | ||
|
|
||
| /** Output names of a raw (Jinja) model's final SELECT; empty when a Jinja | ||
| * expression or tag is in the list. */ | ||
| function rawSelectList(sql: string): Set<string> { | ||
| const plain = sql | ||
| .replace(/\{#[\s\S]*?#\}/g, " ") | ||
| // A tag in the list (`{% if %}` / `{% for %}`) makes its columns conditional: | ||
| // it becomes its own `__jinja__` item, which marks the list unknown. | ||
| .replace(/\{%[\s\S]*?%\}/g, " , __jinja__ , ") | ||
| .replace(/\{\{[\s\S]*?\}\}/g, " __jinja__ ") | ||
| const names = selectListNames(plain) | ||
| if (!names || names.includes("__jinja__")) return new Set() | ||
| // A Jinja expression inside an item (e.g. `{{ col }} as x`) leaves its alias intact. | ||
| return new Set(names) | ||
| } | ||
|
|
||
| const ticks = (xs: string[]) => xs.map((x) => `\`${x}\``).join(", ") | ||
|
|
@@ -1725,6 +1751,9 @@ export async function runReview(input: OrchestrateInput): Promise<VerdictEnvelop | |
| // Fetches run in parallel across schema files (a schema-heavy PR could touch | ||
| // dozens of yml files; serial `git show` per file adds up). | ||
| const schemaFiles = reviewable.filter((f) => f.kind === "schema_yml") | ||
| // The head's dbt_project.yml (test thresholds inherited by new tests), read once. | ||
| const projectYmlPath = `${input.layout?.root ? `${input.layout.root.replace(/\/$/, "")}/` : ""}dbt_project.yml` | ||
| const projectYml = schemaFiles.length ? await getContent?.(projectYmlPath, "new").catch(() => undefined) : undefined | ||
| if (schemaFiles.length) { | ||
| const schemaFindingSets = await Promise.all( | ||
| schemaFiles.map(async (file) => { | ||
|
|
@@ -1738,6 +1767,7 @@ export async function runReview(input: OrchestrateInput): Promise<VerdictEnvelop | |
| ...detectSchemaYmlPatterns(file, input.rubric, { oldContent, newContent }), | ||
| ...detectUnitTestFixtures(file, oldContent, newContent), | ||
| ...snapshotYmlFindings(file, oldContent, newContent), | ||
| ...detectTestAdequacy({ file, oldContent, newContent, projectYml, layout: input.layout }), | ||
| ] | ||
| } catch (err) { | ||
| // One malformed schema file degrades that file only. | ||
|
|
@@ -1748,6 +1778,12 @@ export async function runReview(input: OrchestrateInput): Promise<VerdictEnvelop | |
| ) | ||
| for (const findings of schemaFindingSets) all.push(findings) | ||
| } | ||
| // A dbt_project.yml change that loosens test thresholds for a whole path. | ||
| for (const file of reviewable.filter((f) => f.kind === "project_config" && /(^|\/)dbt_project\.ya?ml$/.test(f.path))) { | ||
| if (file.status !== "modified") continue | ||
| const [oldContent, newContent] = await Promise.all([getContent?.(file.path, "old"), getContent?.(file.path, "new")]) | ||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. SUGGESTION: Re-fetch of When the PR modifies
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Follow-up: low priority, tracked. We'll reuse |
||
| all.push(detectThresholdLoosening(file, oldContent, newContent)) | ||
| } | ||
|
|
||
| // Architectural dedup: the regex `dbt-patterns`/`rule-catalog` layer is a | ||
| // diff-aware, Jinja-aware, offline fallback. For the STRUCTURAL SQL checks | ||
|
|
||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
P3: This snippet pins the action to
@main, but the rest of the page pins@v0.8.5. Pin to the same release tag so users get a reviewed, reproducible action version.Prompt for AI agents
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Not changing yet:
@mainis intentional until a release that includesdbt_prepareships, and we'll re-pin it then.