From de458ec2b88018a41407dd7beec13b0e884aea11 Mon Sep 17 00:00:00 2001 From: RaghwendraSingh Date: Fri, 13 Nov 2026 13:23:09 +0530 Subject: [PATCH 1/4] feat(review): optional `dbt_prepare` in the action builds the head, merge-base and catalog artifacts - new action inputs: `dbt_prepare` (`none` default | `compile` | `full`), `dbt_command`, `dbt_profiles_dir` - `compile` runs `dbt compile` on the head and compiles the PR's merge-base into `target-base/` in a temporary worktree (fetching and deepening a shallow checkout when needed); `full` adds `dbt docs generate` - a base compile that fails is a warning: the review still runs and marks those models unverified - inputs reach the script only through env, never interpolated - docs: the manual base-compile snippet now compiles the merge-base, not the base branch's tip - the example workflow is renamed `examples/dbt-pr-review.yml` (was named after one internal repo) Co-Authored-By: Claude Opus 5.5 --- docs/docs/usage/dbt-pr-review.md | 25 ++++++-- github/review/action.yml | 59 +++++++++++++++++++ ...timate-ingestion.yml => dbt-pr-review.yml} | 2 +- .../test/altimate/review-action.test.ts | 24 ++++++++ .../skill/release-v0.8.5-adversarial.test.ts | 2 +- 5 files changed, 106 insertions(+), 6 deletions(-) rename github/review/examples/{altimate-ingestion.yml => dbt-pr-review.yml} (97%) create mode 100644 packages/opencode/test/altimate/review-action.test.ts diff --git a/docs/docs/usage/dbt-pr-review.md b/docs/docs/usage/dbt-pr-review.md index 1513e7c090..90a8ed0a32 100644 --- a/docs/docs/usage/dbt-pr-review.md +++ b/docs/docs/usage/dbt-pr-review.md @@ -209,13 +209,30 @@ jobs: | `target-base/compiled/…` | Compile the base ref into `target-base/` (below) | Equivalence proofs and column-breakage for modified models. | | `target/catalog.json` | `dbt docs generate` | Complete warehouse columns for column-lineage breakage. | -Compiling the base (optional, for equivalence and column-breakage): +The action can build these for you with `dbt_prepare` (off by default): + +```yaml + - uses: AltimateAI/altimate-code/github/review@main + with: + dbt_prepare: compile # none | compile | full + dbt_command: dbt # or "uv run dbt", "poetry run dbt" + dbt_profiles_dir: . # relative to project_dir +``` + +- **`compile`:** runs `dbt compile` on the head, then compiles the PR's merge-base + into `target-base/` in a temporary worktree. This enables equivalence and + column-breakage proofs. It usually works offline. +- **`full`:** also runs `dbt docs generate` for `catalog.json` (complete column + types). That needs warehouse credentials in the profile. +- A base compile that fails is a warning: the review still runs, and the + affected models are listed as not verified. + +To build them yourself, compile the **merge-base**, not the base branch's tip: ```yaml - name: Compile the base into target-base/ run: | - # The review diffs against the merge-base, not the target branch tip. - git worktree add ../base "$(git merge-base "origin/${{ github.base_ref }}" HEAD)" + git worktree add ../base "$(git merge-base origin/${{ github.base_ref }} HEAD)" (cd ../base && dbt deps && dbt compile --target-path "$GITHUB_WORKSPACE/target-base") ``` @@ -246,7 +263,7 @@ ways, or neither (it self-disables, leaving a deterministic-only review): Always pass keys as repo **secrets**. Warehouse credentials are consumed by the `dbt compile` step (via your `profiles.yml`), **not** by the review step. A complete, copy-paste workflow lives at -[`github/review/examples/altimate-ingestion.yml`](https://github.com/AltimateAI/altimate-code/blob/main/github/review/examples/altimate-ingestion.yml). +[`github/review/examples/dbt-pr-review.yml`](https://github.com/AltimateAI/altimate-code/blob/main/github/review/examples/dbt-pr-review.yml). Re-pushing commits updates the same summary comment in place; fixed findings are dropped on the next run. `--post` targets **GitHub** PRs (it reads `GITHUB_TOKEN` / diff --git a/github/review/action.yml b/github/review/action.yml index 09c4bc1538..f39818fc6b 100644 --- a/github/review/action.yml +++ b/github/review/action.yml @@ -31,6 +31,17 @@ inputs: description: "Post the verdict to the PR (summary comment + inline review)." required: false default: "true" + dbt_prepare: + description: "Build the dbt artifacts the full review uses before reviewing. `none` (default): you build them in earlier steps. `compile`: `dbt compile` on the head, and on the PR's merge-base into `target-base/` (equivalence and lineage proofs). `full`: also `dbt docs generate` for `catalog.json` (complete column types). `compile` usually works offline; `full` needs warehouse credentials in the profile." + required: false + default: "none" + dbt_command: + description: "How to invoke dbt for `dbt_prepare`, e.g. `dbt`, `uv run dbt`, `poetry run dbt`." + required: false + default: "dbt" + dbt_profiles_dir: + description: "`--profiles-dir` for `dbt_prepare`, relative to `project_dir`. Empty = dbt's default lookup." + required: false signing_key: description: "Optional HMAC key to sign the verdict envelope (ALTIMATE_REVIEW_SIGNING_KEY)." required: false @@ -167,6 +178,54 @@ runs: echo "Advisory lane: no model/credentials provided — running deterministic-only." fi + # Optional: build the head and base artifacts the full-tier review needs. + # Inputs are passed as ENV, never interpolated into the script. + - name: Prepare dbt artifacts + if: inputs.dbt_prepare != 'none' + shell: bash + env: + IN_PREPARE: ${{ inputs.dbt_prepare }} + IN_DBT: ${{ inputs.dbt_command }} + IN_PROFILES: ${{ inputs.dbt_profiles_dir }} + IN_PROJECT_DIR: ${{ inputs.project_dir }} + IN_BASE: ${{ inputs.base }} + EVENT_BASE: ${{ github.event.pull_request.base.ref }} + run: | + set -euo pipefail + case "$IN_PREPARE" in compile|full) ;; *) echo "::error::dbt_prepare must be none, compile or full (got '$IN_PREPARE')"; exit 1 ;; esac + read -ra DBT <<< "$IN_DBT" + PROJECT="$(cd "${IN_PROJECT_DIR:-.}" && pwd)" + REL="$(git -C "$PROJECT" rev-parse --show-prefix)" + PROFILES=() + [[ -n "${IN_PROFILES:-}" ]] && PROFILES=(--profiles-dir "$(cd "$PROJECT" && cd "$IN_PROFILES" && pwd)") + build() { # $1 = project dir, $2 = target path + ( cd "$1" + [[ -f packages.yml || -f dependencies.yml ]] && "${DBT[@]}" deps ${PROFILES[@]+"${PROFILES[@]}"} + "${DBT[@]}" compile --target-path "$2" ${PROFILES[@]+"${PROFILES[@]}"} ) + } + echo "Compiling the head…" + build "$PROJECT" "$PROJECT/target" + if [[ "$IN_PREPARE" == "full" ]]; then + echo "Generating catalog.json…" + ( cd "$PROJECT" && "${DBT[@]}" docs generate --target-path "$PROJECT/target" ${PROFILES[@]+"${PROFILES[@]}"} ) + fi + # The base side: the PR's merge-base, compiled into target-base/. + BASE="${IN_BASE:-$EVENT_BASE}" + BASE="${BASE#origin/}" + if [[ -z "$BASE" ]]; then echo "::warning::No base branch known; skipping the base compile."; exit 0; fi + git fetch --no-tags origin "+refs/heads/$BASE:refs/remotes/origin/$BASE" >/dev/null 2>&1 || true + MB="$(git merge-base "origin/$BASE" HEAD 2>/dev/null || git merge-base "$BASE" HEAD 2>/dev/null || true)" + if [[ -z "$MB" && "$(git rev-parse --is-shallow-repository)" == "true" ]]; then + git fetch --no-tags --deepen=1000 origin "+refs/heads/$BASE:refs/remotes/origin/$BASE" "$(git rev-parse HEAD)" >/dev/null 2>&1 || true + MB="$(git merge-base "origin/$BASE" HEAD 2>/dev/null || true)" + fi + if [[ -z "$MB" ]]; then echo "::warning::Could not find the merge-base with $BASE; skipping the base compile."; exit 0; fi + WT="$RUNNER_TEMP/altimate-review-base" + git worktree add --detach "$WT" "$MB" >/dev/null + trap 'git worktree remove --force "$WT" >/dev/null 2>&1 || true' EXIT + echo "Compiling the base ($BASE @ ${MB:0:12})…" + build "$WT/$REL" "$PROJECT/target-base" || echo "::warning::The base compile failed; equivalence will be marked unverified." + - name: Run dbt PR review shell: bash # Inputs are passed as ENV (data), never interpolated into the command — diff --git a/github/review/examples/altimate-ingestion.yml b/github/review/examples/dbt-pr-review.yml similarity index 97% rename from github/review/examples/altimate-ingestion.yml rename to github/review/examples/dbt-pr-review.yml index dc52867909..799d424424 100644 --- a/github/review/examples/altimate-ingestion.yml +++ b/github/review/examples/dbt-pr-review.yml @@ -1,4 +1,4 @@ -# Example: drop-in dbt PR review for a dbt repo (e.g. altimate-ingestion). +# Example: drop-in dbt PR review for a dbt repo. # Copy to the target repo as `.github/workflows/dbt-pr-review.yml` and adjust the # dbt adapter + profiles/secrets to match the warehouse. # diff --git a/packages/opencode/test/altimate/review-action.test.ts b/packages/opencode/test/altimate/review-action.test.ts new file mode 100644 index 0000000000..36f0417224 --- /dev/null +++ b/packages/opencode/test/altimate/review-action.test.ts @@ -0,0 +1,24 @@ +import { describe, test, expect } from "bun:test" +import { promises as fs } from "node:fs" +import path from "node:path" +import YAML from "yaml" + +/** The GitHub Action's optional artifact preparation stays opt-in and injection-safe. */ +const actionPath = path.join(import.meta.dir, "../../../../github/review/action.yml") + +describe("action: dbt_prepare", () => { + test("is opt-in, and its script reads inputs only from env", async () => { + const action = YAML.parse(await fs.readFile(actionPath, "utf8")) + expect(action.inputs.dbt_prepare.default).toBe("none") + expect(action.inputs.dbt_command.default).toBe("dbt") + const steps: Array<{ name: string; if?: string; run?: string; env?: Record }> = action.runs.steps + const prep = steps.find((s) => s.name === "Prepare dbt artifacts")! + const review = steps.findIndex((s) => s.name === "Run dbt PR review") + expect(steps.indexOf(prep)).toBeLessThan(review) + expect(prep.if).toBe("inputs.dbt_prepare != 'none'") + expect(prep.run).not.toContain("${{") + expect(prep.env?.IN_DBT).toBe("${{ inputs.dbt_command }}") + // The base is compiled into target-base/, where the compiled-SQL resolver looks. + expect(prep.run).toContain('"$PROJECT/target-base"') + }) +}) diff --git a/packages/opencode/test/skill/release-v0.8.5-adversarial.test.ts b/packages/opencode/test/skill/release-v0.8.5-adversarial.test.ts index 00b1ee1724..77d91cbf04 100644 --- a/packages/opencode/test/skill/release-v0.8.5-adversarial.test.ts +++ b/packages/opencode/test/skill/release-v0.8.5-adversarial.test.ts @@ -238,7 +238,7 @@ describe("v0.8.5 adversarial - composite action", () => { const changelog = await fs.readFile(path.join(repoRoot, "CHANGELOG.md"), "utf8") expect(changelog).toContain(`## [${version}]`) - for (const relative of ["docs/docs/usage/dbt-pr-review.md", "github/review/examples/altimate-ingestion.yml"]) { + for (const relative of ["docs/docs/usage/dbt-pr-review.md", "github/review/examples/dbt-pr-review.yml"]) { const content = await fs.readFile(path.join(repoRoot, relative), "utf8") expect(content).toContain(`AltimateAI/altimate-code/github/review@v${version}`) expect(content).not.toContain("AltimateAI/altimate-code/github/review@v0.8.4") From e1dfd696bb45fc7d553833f3ee9ab1e6b90b45f6 Mon Sep 17 00:00:00 2001 From: RaghwendraSingh Date: Fri, 13 Nov 2026 13:29:00 +0530 Subject: [PATCH 2/4] fix(review): column-set rules work in `dbt parse`-only CI; dedup windows split across lines are read whole MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Found by replaying the PRs behind known escaped defects: - without compiled SQL, output columns come from the raw model's final SELECT (Jinja blanked; a Jinja expression in the list makes it unknown). `on_schema_change='fail'` with new columns now fires on the two real escapes it targets, in a parse-only CI setup - `row_number() over (…)` windows are read from the whole file, so an `ORDER BY` on its own line is seen: the "dedup has no ORDER BY" rule no longer fires falsely on multi-line windows, and the load-timestamp rule sees them. A window counts only when an added line falls inside it Co-Authored-By: Claude Opus 5.5 --- .../src/altimate/review/dbt-patterns.ts | 44 ++++++++++++---- .../src/altimate/review/orchestrate.ts | 46 +++++++++++----- .../review-incremental-semantics.test.ts | 52 +++++++++++++++++++ 3 files changed, 120 insertions(+), 22 deletions(-) diff --git a/packages/opencode/src/altimate/review/dbt-patterns.ts b/packages/opencode/src/altimate/review/dbt-patterns.ts index 91053d7515..f96eca2649 100644 --- a/packages/opencode/src/altimate/review/dbt-patterns.ts +++ b/packages/opencode/src/altimate/review/dbt-patterns.ts @@ -225,15 +225,42 @@ const detectNotIn: Detector = (c) => { }) } +/** + * `row_number() over (…)` windows in the model, read from the whole file so a + * window split across lines is seen whole. A window counts as part of the + * change when one of the PR's added lines falls inside it. + */ +function dedupWindows(c: Ctx): Array<{ spec: string; text: string }> { + const sql = stripComments(c.newSql) + const added = c.added.map((l) => stripComments(l).trim()).filter((l) => l.length >= 3) + const out: Array<{ spec: string; text: string }> = [] + for (const m of sql.matchAll(/row_number\s*\(\s*\)\s*over\s*\(/gi)) { + const open = m.index! + m[0].length - 1 + let depth = 0 + let close = -1 + for (let i = open; i < sql.length; i++) { + if (sql[i] === "(") depth++ + else if (sql[i] === ")" && --depth === 0) { + close = i + break + } + } + if (close < 0) continue + const text = sql.slice(m.index!, close + 1) + if (!added.some((l) => text.includes(l) || l.includes(text.replace(/\s+/g, " ")))) continue + out.push({ spec: sql.slice(open + 1, close).replace(/\s+/g, " ").trim(), text: text.replace(/\s+/g, " ") }) + } + return out +} + // 9. Dedup via ROW_NUMBER()/QUALIFY whose ORDER BY has no unique tiebreaker. const detectDedupTie: Detector = (c) => { - const hit = c.added.find((l) => /row_number\s*\(\s*\)\s*over\s*\(/i.test(stripComments(l))) - if (!hit) return null // Only flag dedup with NO ORDER BY — that is unambiguously non-deterministic. // A present ORDER BY (even single-column) is the developer's deterministic // choice and the dbt-recommended pattern; warning "it might not be unique" // on every dedup is noise (false positives on the common, correct case). - if (/order\s+by/i.test(stripComments(hit))) return null + const hit = dedupWindows(c).find((w) => !/order\s+by/i.test(w.spec)) + if (!hit) return null return makeFinding({ severity: "warning", category: "dedup", @@ -244,7 +271,7 @@ const detectDedupTie: Detector = (c) => { file: c.file.path, model: c.model, confidence: "high", - evidence: { tool: "dbt-patterns", result: { rule: "dedup_no_tiebreaker", line: hit.trim() } }, + evidence: { tool: "dbt-patterns", result: { rule: "dedup_no_tiebreaker", line: hit.text } }, ruleKey: "dedup:no-tiebreaker", }) } @@ -257,11 +284,8 @@ const detectDedupTie: Detector = (c) => { const LOAD_TS_RE = /^(_?(ingest(ion|ed)?|load(ed)?|etl|extract(ed)?|batch)(_at|_ts|_time|_timestamp|_date|_dt)|_?(ingestion|load|etl|batch)_timestamp|_loaded_at)$/i const detectDedupLoadTimestamp: Detector = (c) => { - for (const line of c.added) { - const s = stripComments(line) - const m = /row_number\s*\(\s*\)\s*over\s*\(([^)]*)\)/i.exec(s) - if (!m) continue - const order = /order\s+by\s+(.+)$/i.exec(m[1])?.[1] + for (const w of dedupWindows(c)) { + const order = /order\s+by\s+(.+)$/i.exec(w.spec)?.[1] if (!order) continue const keys = order.split(",").map((k) => k.trim().replace(/\s+(asc|desc)\b.*$/i, "").replace(/\s+nulls\s+(first|last)$/i, "")) if (keys.length !== 1) continue @@ -277,7 +301,7 @@ const detectDedupLoadTimestamp: Detector = (c) => { file: c.file.path, model: c.model, confidence: "high", - evidence: { tool: "dbt-patterns", result: { rule: "dedup_load_timestamp_only", line: line.trim() } }, + evidence: { tool: "dbt-patterns", result: { rule: "dedup_load_timestamp_only", line: w.text } }, ruleKey: "dedup:load-timestamp-only", }) } diff --git a/packages/opencode/src/altimate/review/orchestrate.ts b/packages/opencode/src/altimate/review/orchestrate.ts index 701118b058..34d108b893 100644 --- a/packages/opencode/src/altimate/review/orchestrate.ts +++ b/packages/opencode/src/altimate/review/orchestrate.ts @@ -661,19 +661,41 @@ async function columnDelta( dialect: string, ): Promise<{ added: string[]; removed: string[]; current: Set }> { const none = { added: [], removed: [], current: new Set() } - const { file, engineNewSql, engineOldSql } = ctx - if (!engineNewSql || file.status === "deleted") return none - const current = await outputColumns(runner, engineNewSql, dialect) - if (!current.size) return none - if (!hasOldSide(file)) return { added: [...current], removed: [], current } - if (!engineOldSql) return none - const before = await outputColumns(runner, engineOldSql, dialect) - if (!before.size) return none - return { - added: [...current].filter((c) => !before.has(c)), - removed: [...before].filter((c) => !current.has(c)), - current, + const { file, engineNewSql, engineOldSql, newSql, oldSql } = ctx + if (file.status === "deleted") return none + const withOld = hasOldSide(file) + const delta = (current: Set, before?: Set) => + !current.size || (withOld && !before?.size) + ? none + : { + added: [...current].filter((c) => !before?.has(c)), + removed: before ? [...before].filter((c) => !current.has(c)) : [], + current, + } + // Compiled SQL on both sides: column lineage plus the final SELECT's names. + if (engineNewSql && (!withOld || engineOldSql)) { + const current = await outputColumns(runner, engineNewSql, dialect) + const before = withOld ? await outputColumns(runner, engineOldSql!, dialect) : undefined + return delta(current, before) } + // No compiled SQL (CI that only runs `dbt parse`): the raw model's own SELECT + // list, when it names its columns. Columns produced inside a macro stay unknown. + if (!newSql) return none + const current = rawSelectList(newSql) + const before = withOld && oldSql ? rawSelectList(oldSql) : undefined + return delta(current, before) +} + +/** Output names of a raw (Jinja) model's final SELECT; empty when a Jinja expression is in the list. */ +function rawSelectList(sql: string): Set { + const plain = sql + .replace(/\{#[\s\S]*?#\}/g, " ") + .replace(/\{%[\s\S]*?%\}/g, " ") + .replace(/\{\{[\s\S]*?\}\}/g, " __jinja__ ") + const names = selectListNames(plain) + if (!names || names.includes("__jinja__")) return new Set() + // A Jinja expression inside an item (e.g. `{{ col }} as x`) leaves its alias intact. + return new Set(names) } const ticks = (xs: string[]) => xs.map((x) => `\`${x}\``).join(", ") diff --git a/packages/opencode/test/altimate/review-incremental-semantics.test.ts b/packages/opencode/test/altimate/review-incremental-semantics.test.ts index bbf424ceb6..5fd78deef1 100644 --- a/packages/opencode/test/altimate/review-incremental-semantics.test.ts +++ b/packages/opencode/test/altimate/review-incremental-semantics.test.ts @@ -299,3 +299,55 @@ describe("end to end through the orchestrator", () => { expect(env.verdict).not.toBe("REQUEST_CHANGES") }) }) + +describe("dedup windows split across lines", () => { + const sql = "select *,\n row_number() over (\n partition by job_id\n order by ingestion_timestamp desc\n ) as rn\nfrom t" + const rulesFor = (diffAdded: string) => + detectModelPatterns({ path: "models/m.sql", status: "modified", diff: diffAdded }, sql, DEFAULT_RUBRIC).map( + (f) => (f.evidence?.result as { rule?: string })?.rule, + ) + test("an ORDER BY on its own line is seen: no false 'no ORDER BY', and the load-timestamp rule fires", () => { + const r = rulesFor("+ row_number() over (\n") + expect(r).not.toContain("dedup_no_tiebreaker") + expect(r).toContain("dedup_load_timestamp_only") + }) + test("a window the PR doesn't touch is not reported", () => { + expect(rulesFor("+select 1 as unrelated\n")).not.toContain("dedup_load_timestamp_only") + }) +}) + +describe("column changes without compiled SQL (dbt parse-only CI)", () => { + test("on_schema_change='fail' sees columns added to the raw model's SELECT", async () => { + const cfg = "{{ config(materialized='incremental', unique_key='id', on_schema_change='fail') }}\n" + const oldSql = `${cfg}select id, amount from {{ ref('src') }}` + const newSql = `${cfg}select id, amount, region from {{ ref('src') }}` + const env = await runReview({ + changedFiles: [{ path: "models/marts/fct.sql", status: "modified", diff: `-${oldSql}\n+${newSql}\n` }], + config: { ...DEFAULT_REVIEW_CONFIG, reviewers: ["dbt_patterns"] }, + rubric: DEFAULT_RUBRIC, + mode: "comment", + runner: { + async impact() { + return { hasManifest: true, severity: "SAFE", directCount: 0, transitiveCount: 0, testCount: 0 } + }, + async grade() { + return {} + }, + async check() { + return { issues: [], ran: true } + }, + async equivalence() { + return { decided: false } + }, + async detectPii() { + return { columns: [] } + }, + async columnLineage() { + return [] + }, + } as ReviewRunner, + getContent: async (_f, side) => (side === "old" ? oldSql : newSql), + }) + expect(env.findings.find((f) => f.title.includes("on_schema_change"))?.body).toContain("adds `region`") + }) +}) From bf316064d94886f9094114bb50acad09d4d9848e Mon Sep 17 00:00:00 2001 From: RaghwendraSingh Date: Fri, 13 Nov 2026 13:32:13 +0530 Subject: [PATCH 3/4] feat(review): data tests that can't do their job: self-comparisons, inherited warn/thresholds, loosened project thresholds MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - a test added or changed in the PR that compares a model with itself (`dbt_utils.equality`/`equal_rowcount`/… against its own `ref`, or a `relationships` test to its own column) passes by construction (warning) - new tests that inherit `severity: warn` or an `error_if` threshold from `dbt_project.yml` (walked from the top through the project name and folders, deeper wins) are reported once per yml (suggestion) - a `dbt_project.yml` change that turns a folder's tests to `warn` or sets an `error_if` threshold is reported (warning) - reads dbt 1.8+ `arguments:` too; all findings are catalog origin (never block) Co-Authored-By: Claude Opus 5.5 --- docs/docs/usage/dbt-pr-review.md | 8 + .../src/altimate/review/orchestrate.ts | 11 + .../src/altimate/review/test-adequacy.ts | 214 ++++++++++++++++++ .../altimate/review-test-adequacy.test.ts | 83 +++++++ 4 files changed, 316 insertions(+) create mode 100644 packages/opencode/src/altimate/review/test-adequacy.ts create mode 100644 packages/opencode/test/altimate/review-test-adequacy.test.ts diff --git a/docs/docs/usage/dbt-pr-review.md b/docs/docs/usage/dbt-pr-review.md index 90a8ed0a32..5d07e23204 100644 --- a/docs/docs/usage/dbt-pr-review.md +++ b/docs/docs/usage/dbt-pr-review.md @@ -517,6 +517,14 @@ The deterministic rules group into four families: - **dbt unit tests.** A unit test whose `given` inputs are all `rows: []` is reported: it either never tests the model's logic, or expects rows the model has nothing to build from. +- **Data tests that can't do their job.** These are checked on tests added or + changed in the PR: + - a test that compares a model with itself (`dbt_utils.equality` or + `equal_rowcount` against its own `ref`, or a `relationships` test pointing + at its own column) passes by construction; + - new tests that inherit `severity: warn` or an `error_if` threshold from + `dbt_project.yml` report failures without failing the build; + - a `dbt_project.yml` change that loosens those thresholds for a whole folder. - **Incremental and snapshot semantics.** Config is read from the manifest (project, yml and inline, as dbt merges them) with the head's inline `{{ config() }}` on top. Formatting-only edits never count as a change. diff --git a/packages/opencode/src/altimate/review/orchestrate.ts b/packages/opencode/src/altimate/review/orchestrate.ts index 34d108b893..d1edae1f25 100644 --- a/packages/opencode/src/altimate/review/orchestrate.ts +++ b/packages/opencode/src/altimate/review/orchestrate.ts @@ -26,6 +26,7 @@ import { detectModelPatterns, detectSchemaYmlPatterns, splitDiff } from "./dbt-p import { listMirrorsModel, type ConsumerColumnList } from "./consumer-lists" import { detectUnitTestFixtures } from "./unit-tests" import { selectListNames } from "./select-list" +import { detectTestAdequacy, detectThresholdLoosening } from "./test-adequacy" import { incrementalFindings, snapshotFindings, snapshotYmlFindings } from "./incremental-semantics" import { type AiLaneStatus, type AiReviewInput, type AiReviewResult, safeReason } from "./ai-review" @@ -1747,6 +1748,9 @@ export async function runReview(input: OrchestrateInput): Promise f.kind === "schema_yml") + // The head's dbt_project.yml (test thresholds inherited by new tests), read once. + const projectYmlPath = `${input.layout?.root ? `${input.layout.root.replace(/\/$/, "")}/` : ""}dbt_project.yml` + const projectYml = schemaFiles.length ? await getContent?.(projectYmlPath, "new").catch(() => undefined) : undefined if (schemaFiles.length) { const schemaFindingSets = await Promise.all( schemaFiles.map(async (file) => { @@ -1760,6 +1764,7 @@ export async function runReview(input: OrchestrateInput): Promise f.kind === "project_config" && /(^|\/)dbt_project\.ya?ml$/.test(f.path))) { + if (file.status !== "modified") continue + const [oldContent, newContent] = await Promise.all([getContent?.(file.path, "old"), getContent?.(file.path, "new")]) + all.push(detectThresholdLoosening(file, oldContent, newContent)) + } // Architectural dedup: the regex `dbt-patterns`/`rule-catalog` layer is a // diff-aware, Jinja-aware, offline fallback. For the STRUCTURAL SQL checks diff --git a/packages/opencode/src/altimate/review/test-adequacy.ts b/packages/opencode/src/altimate/review/test-adequacy.ts new file mode 100644 index 0000000000..65fb2848cc --- /dev/null +++ b/packages/opencode/src/altimate/review/test-adequacy.ts @@ -0,0 +1,214 @@ +import YAML from "yaml" +import { makeFinding, type Finding } from "./finding" +import type { ChangedFile, DbtProjectLayout } from "./diff-filter" + +/** + * Data tests that cannot do their job, found from yml alone (any adapter): + * - a test that compares a model with itself (`dbt_utils.equality` / + * `equal_rowcount` against its own `ref`, or a `relationships` test pointing + * at its own column) passes by construction; + * - tests added in the PR that inherit `severity: warn` (or an `error_if` + * threshold) from `dbt_project.yml` report failures without failing the build; + * - a PR that loosens those thresholds in `dbt_project.yml` weakens every test + * under that path. + * Only tests added or changed in the PR are checked. + */ + +interface TestRef { + entity: string + column?: string + test: string + args: Record +} + +const SELF_COMPARE = new Set([ + "equality", + "equal_rowcount", + "cardinality_equality", + "fewer_rows_than", + "expect_table_row_count_to_equal_other_table", + "expect_table_aggregation_to_equal_other_table", +]) + +function parse(text?: string): any { + if (!text) return undefined + try { + return YAML.parse(text) + } catch { + return undefined + } +} + +function testsOf(doc: any): TestRef[] { + const out: TestRef[] = [] + const read = (list: unknown, entity: string, column?: string) => { + if (!Array.isArray(list)) return + for (const t of list) { + if (typeof t === "string") out.push({ entity, column, test: t, args: {} }) + else if (t && typeof t === "object") { + const [k] = Object.keys(t) + if (k) out.push({ entity, column, test: k, args: (t as any)[k] ?? {} }) + } + } + } + for (const group of ["models", "snapshots", "seeds"]) { + for (const e of Array.isArray(doc?.[group]) ? doc[group] : []) { + if (typeof e?.name !== "string") continue + read(e.data_tests ?? e.tests, e.name) + for (const c of Array.isArray(e.columns) ? e.columns : []) + if (typeof c?.name === "string") read(c.data_tests ?? c.tests, e.name, c.name) + } + } + return out +} + +const key = (t: TestRef) => `${t.entity}|${t.column ?? ""}|${t.test}|${JSON.stringify(t.args)}` +const arg = (t: TestRef, name: string) => t.args?.[name] ?? t.args?.arguments?.[name] +const shortName = (test: string) => test.split(".").pop()!.toLowerCase() +const refsEntity = (v: unknown, entity: string) => + typeof v === "string" && new RegExp(`ref\\(\\s*['"]${entity.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")}['"]\\s*\\)`).test(v) + +function finding(file: ChangedFile, model: string | undefined, rule: string, severity: Finding["severity"], title: string, body: string, extra = ""): Finding { + return makeFinding({ + severity, + category: "test_coverage", + title, + body, + file: file.path, + model, + confidence: "high", + evidence: { tool: "test-adequacy", result: { rule } }, + ruleKey: `test_coverage:${rule}${extra}`, + }) +} + +/** `+severity` / `+error_if` / `+warn_if` (with or without `+`) on one config node. */ +function thresholds(node: any): { severity?: string; error_if?: string; warn_if?: string } { + if (!node || typeof node !== "object") return {} + const get = (k: string) => (node[`+${k}`] ?? node[k]) as unknown + const str = (v: unknown) => (typeof v === "string" ? v.trim().toLowerCase() : undefined) + return { severity: str(get("severity")), error_if: str(get("error_if")), warn_if: str(get("warn_if")) } +} + +/** + * Effective test thresholds for a yml under `segments` (folders below the model + * path), walking `data_tests:` (or `tests:`) from the top, through the project + * name, down the folders; deeper levels override. `path` is where the effective + * value was set. + */ +export function inheritedThresholds(projectYml: any, segments: string[]): { severity?: string; error_if?: string; path: string[] } { + const tree = projectYml?.data_tests ?? projectYml?.tests + if (!tree || typeof tree !== "object") return { path: [] } + const name = projectYml?.name + const chain = name && tree[name] && typeof tree[name] === "object" ? [String(name), ...segments] : segments + const levels: Array<[string[], any]> = [[[], tree]] + let node = tree + for (let i = 0; i < chain.length; i++) { + node = node?.[chain[i]] + if (!node || typeof node !== "object") break + levels.push([chain.slice(0, i + 1), node]) + } + const eff: { severity?: string; error_if?: string } = {} + let path: string[] = [] + for (const [at, n] of levels) { + const t = thresholds(n) + if (t.severity) eff.severity = t.severity + if (t.error_if) eff.error_if = t.error_if + if (t.severity || t.error_if) path = at.length ? at : ["(all tests)"] + } + return { ...eff, path } +} + +/** Folders of a yml below its model path (e.g. `models/marts/x.yml` → `["marts"]`). */ +function segmentsOf(file: string, layout?: DbtProjectLayout): string[] { + const root = layout?.root ? layout.root.replace(/\/$/, "") + "/" : "" + const rel = root && file.startsWith(root) ? file.slice(root.length) : file + const parts = rel.split("/").slice(0, -1) + const modelDirs = layout?.dirs.model ?? ["models"] + return modelDirs.includes(parts[0]) ? parts.slice(1) : parts +} + +export function detectTestAdequacy(input: { + file: ChangedFile + oldContent?: string + newContent?: string + projectYml?: string + layout?: DbtProjectLayout +}): Finding[] { + const { file } = input + if (file.status === "deleted") return [] + const newDoc = parse(input.newContent) + if (!newDoc) return [] + const before = new Set(testsOf(parse(input.oldContent)).map(key)) + const added = testsOf(newDoc).filter((t) => !before.has(key(t))) + const out: Finding[] = [] + + for (const t of added) { + const name = shortName(t.test) + const self = + (SELF_COMPARE.has(name) && ["compare_model", "to", "compare_to"].some((a) => refsEntity(arg(t, a), t.entity))) || + (name === "relationships" && refsEntity(arg(t, "to"), t.entity) && String(arg(t, "field") ?? "").toLowerCase() === (t.column ?? "").toLowerCase()) + if (self) + out.push( + finding(file, t.entity, "test-compares-model-with-itself", "warning", `${t.entity}: \`${t.test}\` compares the model with itself`, + `This test checks \`${t.entity}\`${t.column ? `.\`${t.column}\`` : ""} against \`ref('${t.entity}')\`, so it passes by construction. ` + + "Point it at the independent source of truth it is meant to reconcile against.", `:${key(t)}`), + ) + } + + const project = parse(input.projectYml) + if (project && added.length) { + const inherited = inheritedThresholds(project, segmentsOf(file.path, input.layout)) + const own = (t: TestRef) => { + const c = t.args?.config ?? {} + return [c.severity, c.error_if, c.warn_if, t.args?.severity, t.args?.error_if].some((v) => v !== undefined) + } + const loose = added.filter((t) => !own(t)) + const where = inherited.path.length ? ` (\`data_tests:\` → ${inherited.path.map((p) => `\`${p}\``).join(" → ")})` : "" + if (loose.length && inherited.severity === "warn") + out.push( + finding(file, undefined, "test-inherits-warn", "suggestion", `${loose.length} new test(s) in ${file.path} inherit \`severity: warn\``, + `They inherit \`severity: warn\` from \`dbt_project.yml\`${where}, so failures are reported but never fail \`dbt build\`. ` + + "If a test should gate the build, set `config: {severity: error}` on it."), + ) + else if (loose.length && inherited.error_if && inherited.error_if !== "!= 0") + out.push( + finding(file, undefined, "test-inherits-threshold", "suggestion", `${loose.length} new test(s) in ${file.path} inherit \`error_if: ${inherited.error_if}\``, + `They fail only when \`${inherited.error_if}\` rows fail, a threshold inherited from \`dbt_project.yml\`${where}. ` + + "Set `config: {error_if: '!= 0'}` on tests that must catch any bad row."), + ) + } + return out +} + +/** A `dbt_project.yml` change that loosens test thresholds for a whole path. */ +export function detectThresholdLoosening(file: ChangedFile, oldContent?: string, newContent?: string): Finding[] { + const oldDoc = parse(oldContent) + const newDoc = parse(newContent) + if (!oldDoc || !newDoc) return [] + const flat = (doc: any) => { + const out = new Map>() + const visit = (node: any, path: string[]) => { + if (!node || typeof node !== "object") return + out.set(path.join(" → ") || "(all tests)", thresholds(node)) + for (const [k, v] of Object.entries(node)) if (!k.startsWith("+") && v && typeof v === "object") visit(v, [...path, k]) + } + visit(doc?.data_tests ?? doc?.tests, []) + return out + } + const before = flat(oldDoc) + const out: Finding[] = [] + for (const [path, t] of flat(newDoc)) { + const prev = before.get(path) ?? {} + const toWarn = t.severity === "warn" && prev.severity !== "warn" + const newThreshold = t.error_if && t.error_if !== "!= 0" && t.error_if !== prev.error_if + if (!toWarn && !newThreshold) continue + out.push( + finding(file, undefined, "test-thresholds-loosened", "warning", `test thresholds loosened for \`${path}\``, + (toWarn ? "`severity: warn` now applies" : `\`error_if: ${t.error_if}\` now applies`) + + ` to every data test under \`${path}\` in \`dbt_project.yml\`, so those tests stop failing the build. ` + + "Prefer setting it on the specific tests that need it.", `:${path}`), + ) + } + return out +} diff --git a/packages/opencode/test/altimate/review-test-adequacy.test.ts b/packages/opencode/test/altimate/review-test-adequacy.test.ts new file mode 100644 index 0000000000..644e664d14 --- /dev/null +++ b/packages/opencode/test/altimate/review-test-adequacy.test.ts @@ -0,0 +1,83 @@ +import { describe, test, expect } from "bun:test" +import YAML from "yaml" +import { detectTestAdequacy, detectThresholdLoosening, inheritedThresholds } from "../../src/altimate/review/test-adequacy" + +/** Data tests that can't do their job, from yml alone (any adapter). */ + +const file = { path: "models/marts/_fct.yml", status: "modified" as const } +const rule = (fs: { evidence?: { result?: unknown } }[]) => fs.map((f) => (f.evidence?.result as { rule: string }).rule) + +describe("tests that compare a model with itself", () => { + const yml = (compare: string) => `models: + - name: fct_orders + data_tests: + - dbt_utils.equal_rowcount: + compare_model: ${compare} + columns: + - name: customer_id + data_tests: + - relationships: + to: ref('fct_orders') + field: customer_id +` + test("own-ref comparisons and self relationships pass by construction", () => { + const out = detectTestAdequacy({ file, newContent: yml("ref('fct_orders')") }) + expect(rule(out)).toEqual(["test-compares-model-with-itself", "test-compares-model-with-itself"]) + }) + + test("a comparison with another model is fine; unchanged tests are skipped", () => { + const other = yml("ref('stg_orders')").replace("to: ref('fct_orders')", "to: ref('dim_customers')") + expect(detectTestAdequacy({ file, newContent: other })).toEqual([]) + expect(detectTestAdequacy({ file, oldContent: yml("ref('fct_orders')"), newContent: yml("ref('fct_orders')") })).toEqual([]) + }) + + test("dbt 1.8+ `arguments:` form is read too", () => { + const y = `models:\n - name: m\n data_tests:\n - dbt_utils.equality:\n arguments:\n compare_model: ref("m")\n` + expect(rule(detectTestAdequacy({ file: { path: "models/_m.yml", status: "added" }, newContent: y }))).toEqual(["test-compares-model-with-itself"]) + }) +}) + +describe("thresholds inherited from dbt_project.yml", () => { + const project = `name: shop +data_tests: + shop: + marts: + +severity: warn + staging: + +error_if: ">100" +` + const newTests = (own = "") => `models: + - name: fct_orders + columns: + - name: id + data_tests: + - not_null${own} +` + test("the effective value walks project → folders, deeper wins", () => { + const doc = YAML.parse(project) + expect(inheritedThresholds(doc, ["marts"])).toEqual({ severity: "warn", path: ["shop", "marts"] }) + expect(inheritedThresholds(doc, ["staging", "x"])).toEqual({ error_if: ">100", path: ["shop", "staging"] }) + expect(inheritedThresholds(doc, ["core"])).toEqual({ path: [] }) + }) + + test("new tests under a warn folder are reported; a test with its own severity is not", () => { + const out = detectTestAdequacy({ file, newContent: newTests(), projectYml: project }) + expect(rule(out)).toEqual(["test-inherits-warn"]) + expect(out[0].body).toContain("`shop` → `marts`") + const own = newTests(":\n config:\n severity: error").replace("- not_null:", "- not_null:") + expect(detectTestAdequacy({ file, newContent: own, projectYml: project })).toEqual([]) + }) + + test("an inherited error_if threshold is reported", () => { + const out = detectTestAdequacy({ file: { path: "models/staging/_stg.yml", status: "modified" }, newContent: newTests(), projectYml: project }) + expect(rule(out)).toEqual(["test-inherits-threshold"]) + }) + + test("a dbt_project.yml change that loosens thresholds is reported", () => { + const before = "name: shop\ndata_tests:\n shop:\n marts:\n +severity: error\n" + const after = "name: shop\ndata_tests:\n shop:\n marts:\n +severity: warn\n" + const out = detectThresholdLoosening({ path: "dbt_project.yml", status: "modified" }, before, after) + expect(out.map((f) => f.title)).toEqual(["test thresholds loosened for `shop → marts`"]) + expect(detectThresholdLoosening({ path: "dbt_project.yml", status: "modified" }, after, after)).toEqual([]) + }) +}) From e77d121049fb2d8efe138ab854e2dbbefa43d0b9 Mon Sep 17 00:00:00 2001 From: RaghwendraSingh Date: Fri, 20 Nov 2026 10:28:50 +0530 Subject: [PATCH 4/4] fix(review): address review comments: conditional raw columns, dedup window touch, self-compare tests, `error_if` loosening, prepare-step failures - `orchestrate.ts`: a `{% if %}` / `{% for %}` tag inside a raw model's select list makes the list unknown, so conditional columns no longer read as added or removed in the parse-only column delta - `dbt-patterns.ts`: a dedup window counts as changed only when an added line falls inside its line span (head line numbers from the hunk headers), not when an added line's text happens to occur in it - `test-adequacy.ts`: drop `fewer_rows_than` (it fails against itself, it doesn't pass by construction); an aggregation comparison on the same model is flagged only when both expressions match - `test-adequacy.ts`: `error_if` is reported as loosened only when the new bound tolerates more failing rows; `> 0` / `!=0` / `>= 1` count as the strict default - `action.yml`: a failed `dbt docs generate` or base checkout warns instead of failing the step; a stale `target-base/` is removed before the base compile - tests for each Co-Authored-By: Claude Opus 5.5 --- github/review/action.yml | 7 ++- .../src/altimate/review/dbt-patterns.ts | 34 +++++++++++-- .../src/altimate/review/orchestrate.ts | 7 ++- .../src/altimate/review/test-adequacy.ts | 32 ++++++++++-- .../test/altimate/review-action.test.ts | 9 ++++ .../review-incremental-semantics.test.ts | 50 +++++++++++++++++++ .../altimate/review-test-adequacy.test.ts | 27 ++++++++++ 7 files changed, 155 insertions(+), 11 deletions(-) diff --git a/github/review/action.yml b/github/review/action.yml index f39818fc6b..02f6f7d7e9 100644 --- a/github/review/action.yml +++ b/github/review/action.yml @@ -207,7 +207,8 @@ runs: build "$PROJECT" "$PROJECT/target" if [[ "$IN_PREPARE" == "full" ]]; then echo "Generating catalog.json…" - ( cd "$PROJECT" && "${DBT[@]}" docs generate --target-path "$PROJECT/target" ${PROFILES[@]+"${PROFILES[@]}"} ) + ( cd "$PROJECT" && "${DBT[@]}" docs generate --target-path "$PROJECT/target" ${PROFILES[@]+"${PROFILES[@]}"} ) || + echo "::warning::dbt docs generate failed; continuing without catalog.json." fi # The base side: the PR's merge-base, compiled into target-base/. BASE="${IN_BASE:-$EVENT_BASE}" @@ -221,8 +222,10 @@ runs: fi if [[ -z "$MB" ]]; then echo "::warning::Could not find the merge-base with $BASE; skipping the base compile."; exit 0; fi WT="$RUNNER_TEMP/altimate-review-base" - git worktree add --detach "$WT" "$MB" >/dev/null trap 'git worktree remove --force "$WT" >/dev/null 2>&1 || true' EXIT + git worktree add --detach "$WT" "$MB" >/dev/null || { echo "::warning::Could not check out the base; skipping the base compile."; exit 0; } + # A stale target-base/ from an earlier run must not stand in for a failed compile. + rm -rf "$PROJECT/target-base" echo "Compiling the base ($BASE @ ${MB:0:12})…" build "$WT/$REL" "$PROJECT/target-base" || echo "::warning::The base compile failed; equivalence will be marked unverified." diff --git a/packages/opencode/src/altimate/review/dbt-patterns.ts b/packages/opencode/src/altimate/review/dbt-patterns.ts index f96eca2649..02ceb357f5 100644 --- a/packages/opencode/src/altimate/review/dbt-patterns.ts +++ b/packages/opencode/src/altimate/review/dbt-patterns.ts @@ -225,13 +225,38 @@ const detectNotIn: Detector = (c) => { }) } +/** Head-side line numbers of the added lines; undefined when the diff has no hunk headers. */ +function addedLineNumbers(diff: string | undefined): Set | undefined { + if (!diff || !/^@@ /m.test(diff)) return undefined + const out = new Set() + let right = 0 + let inHunk = false + for (const line of diff.split("\n")) { + const h = /^@@ -\d+(?:,\d+)? \+(\d+)(?:,\d+)? @@/.exec(line) + if (h) { + right = Number(h[1]) + inHunk = true + continue + } + if (!inHunk || line.startsWith("\\")) continue + if (line.startsWith("+")) out.add(right++) + else if (line.startsWith(" ") || line === "") right++ + else if (!line.startsWith("-")) inHunk = false + } + return out +} + /** * `row_number() over (…)` windows in the model, read from the whole file so a * window split across lines is seen whole. A window counts as part of the - * change when one of the PR's added lines falls inside it. + * change when one of the PR's added lines falls inside its line span (by text + * only when the diff carries no hunk headers). */ function dedupWindows(c: Ctx): Array<{ spec: string; text: string }> { - const sql = stripComments(c.newSql) + // Comments blanked with newlines kept, so offsets map to head line numbers. + const sql = c.newSql.replace(/\/\*[\s\S]*?\*\/|--[^\n]*|\{#[\s\S]*?#\}/g, (m) => m.replace(/[^\n]/g, " ")) + const lineAt = (offset: number) => sql.slice(0, offset).split("\n").length + const addedLines = addedLineNumbers(c.file.diff) const added = c.added.map((l) => stripComments(l).trim()).filter((l) => l.length >= 3) const out: Array<{ spec: string; text: string }> = [] for (const m of sql.matchAll(/row_number\s*\(\s*\)\s*over\s*\(/gi)) { @@ -247,7 +272,10 @@ function dedupWindows(c: Ctx): Array<{ spec: string; text: string }> { } if (close < 0) continue const text = sql.slice(m.index!, close + 1) - if (!added.some((l) => text.includes(l) || l.includes(text.replace(/\s+/g, " ")))) continue + const touched = addedLines + ? [...addedLines].some((n) => n >= lineAt(m.index!) && n <= lineAt(close)) + : added.some((l) => text.includes(l) || l.includes(text.replace(/\s+/g, " "))) + if (!touched) continue out.push({ spec: sql.slice(open + 1, close).replace(/\s+/g, " ").trim(), text: text.replace(/\s+/g, " ") }) } return out diff --git a/packages/opencode/src/altimate/review/orchestrate.ts b/packages/opencode/src/altimate/review/orchestrate.ts index d1edae1f25..d1426b4a35 100644 --- a/packages/opencode/src/altimate/review/orchestrate.ts +++ b/packages/opencode/src/altimate/review/orchestrate.ts @@ -687,11 +687,14 @@ async function columnDelta( return delta(current, before) } -/** Output names of a raw (Jinja) model's final SELECT; empty when a Jinja expression is in the list. */ +/** Output names of a raw (Jinja) model's final SELECT; empty when a Jinja + * expression or tag is in the list. */ function rawSelectList(sql: string): Set { const plain = sql .replace(/\{#[\s\S]*?#\}/g, " ") - .replace(/\{%[\s\S]*?%\}/g, " ") + // A tag in the list (`{% if %}` / `{% for %}`) makes its columns conditional: + // it becomes its own `__jinja__` item, which marks the list unknown. + .replace(/\{%[\s\S]*?%\}/g, " , __jinja__ , ") .replace(/\{\{[\s\S]*?\}\}/g, " __jinja__ ") const names = selectListNames(plain) if (!names || names.includes("__jinja__")) return new Set() diff --git a/packages/opencode/src/altimate/review/test-adequacy.ts b/packages/opencode/src/altimate/review/test-adequacy.ts index 65fb2848cc..7d2713719e 100644 --- a/packages/opencode/src/altimate/review/test-adequacy.ts +++ b/packages/opencode/src/altimate/review/test-adequacy.ts @@ -25,7 +25,6 @@ const SELF_COMPARE = new Set([ "equality", "equal_rowcount", "cardinality_equality", - "fewer_rows_than", "expect_table_row_count_to_equal_other_table", "expect_table_aggregation_to_equal_other_table", ]) @@ -82,6 +81,19 @@ function finding(file: ChangedFile, model: string | undefined, rule: string, sev }) } +/** + * Failing rows an `error_if` condition tolerates: `!= 0` / `> 0` → 0, `> 10` → 10, + * `>= 10` → 9. Undefined when the condition isn't a simple bound. + */ +function allowedFailures(cond: string | undefined): number | undefined { + const m = cond ? /^(!=|<>|>=|>)\s*(\d+)$/.exec(cond.trim()) : null + if (!m) return undefined + const n = Number(m[2]) + if (m[1] === ">=") return Math.max(0, n - 1) + if (m[1] === ">") return n + return n === 0 ? 0 : undefined +} + /** `+severity` / `+error_if` / `+warn_if` (with or without `+`) on one config node. */ function thresholds(node: any): { severity?: string; error_if?: string; warn_if?: string } { if (!node || typeof node !== "object") return {} @@ -145,8 +157,14 @@ export function detectTestAdequacy(input: { for (const t of added) { const name = shortName(t.test) + // Different expressions on the same model are a valid invariant (e.g. sum of + // line items equals the order total); only the same expression is circular. + const sameExpression = + name !== "expect_table_aggregation_to_equal_other_table" || + String(arg(t, "compare_expression") ?? arg(t, "expression") ?? "").replace(/\s+/g, "").toLowerCase() === + String(arg(t, "expression") ?? "").replace(/\s+/g, "").toLowerCase() const self = - (SELF_COMPARE.has(name) && ["compare_model", "to", "compare_to"].some((a) => refsEntity(arg(t, a), t.entity))) || + (SELF_COMPARE.has(name) && sameExpression && ["compare_model", "to", "compare_to"].some((a) => refsEntity(arg(t, a), t.entity))) || (name === "relationships" && refsEntity(arg(t, "to"), t.entity) && String(arg(t, "field") ?? "").toLowerCase() === (t.column ?? "").toLowerCase()) if (self) out.push( @@ -171,7 +189,7 @@ export function detectTestAdequacy(input: { `They inherit \`severity: warn\` from \`dbt_project.yml\`${where}, so failures are reported but never fail \`dbt build\`. ` + "If a test should gate the build, set `config: {severity: error}` on it."), ) - else if (loose.length && inherited.error_if && inherited.error_if !== "!= 0") + else if (loose.length && inherited.error_if && allowedFailures(inherited.error_if) !== 0) out.push( finding(file, undefined, "test-inherits-threshold", "suggestion", `${loose.length} new test(s) in ${file.path} inherit \`error_if: ${inherited.error_if}\``, `They fail only when \`${inherited.error_if}\` rows fail, a threshold inherited from \`dbt_project.yml\`${where}. ` + @@ -201,7 +219,13 @@ export function detectThresholdLoosening(file: ChangedFile, oldContent?: string, for (const [path, t] of flat(newDoc)) { const prev = before.get(path) ?? {} const toWarn = t.severity === "warn" && prev.severity !== "warn" - const newThreshold = t.error_if && t.error_if !== "!= 0" && t.error_if !== prev.error_if + // Only a threshold that tolerates more failing rows than before loosens; + // a changed condition that is not a simple bound is still reported. + const allowed = allowedFailures(t.error_if) + const newThreshold = + !!t.error_if && + t.error_if !== prev.error_if && + (allowed === undefined || allowed > (allowedFailures(prev.error_if) ?? 0)) if (!toWarn && !newThreshold) continue out.push( finding(file, undefined, "test-thresholds-loosened", "warning", `test thresholds loosened for \`${path}\``, diff --git a/packages/opencode/test/altimate/review-action.test.ts b/packages/opencode/test/altimate/review-action.test.ts index 36f0417224..fda2d21a26 100644 --- a/packages/opencode/test/altimate/review-action.test.ts +++ b/packages/opencode/test/altimate/review-action.test.ts @@ -21,4 +21,13 @@ describe("action: dbt_prepare", () => { // The base is compiled into target-base/, where the compiled-SQL resolver looks. expect(prep.run).toContain('"$PROJECT/target-base"') }) + + test("a failed docs generate or base checkout warns and lets the review run", async () => { + const action = YAML.parse(await fs.readFile(actionPath, "utf8")) + const prep = (action.runs.steps as Array<{ name: string; run?: string }>).find((s) => s.name === "Prepare dbt artifacts")! + const run = prep.run! + expect(run).toMatch(/docs generate[^\n]*\) \|\|\n\s*echo "::warning::/) + expect(run).toMatch(/git worktree add[^\n]*\|\| \{ echo "::warning::/) + expect(run.indexOf('rm -rf "$PROJECT/target-base"')).toBeLessThan(run.indexOf('build "$WT/$REL"')) + }) }) diff --git a/packages/opencode/test/altimate/review-incremental-semantics.test.ts b/packages/opencode/test/altimate/review-incremental-semantics.test.ts index 5fd78deef1..ffddb310b7 100644 --- a/packages/opencode/test/altimate/review-incremental-semantics.test.ts +++ b/packages/opencode/test/altimate/review-incremental-semantics.test.ts @@ -314,6 +314,19 @@ describe("dedup windows split across lines", () => { test("a window the PR doesn't touch is not reported", () => { expect(rulesFor("+select 1 as unrelated\n")).not.toContain("dedup_load_timestamp_only") }) + test("with hunk headers, an added line matching window text elsewhere doesn't touch the window", () => { + const noOrder = + "select\n tenant\n , row_number() over (partition by tenant, id) as rn\nfrom t\nunion all\nselect\n tenant\n , id\nfrom u" + const rules = (diff: string) => + detectModelPatterns({ path: "models/m.sql", status: "modified", diff }, noOrder, DEFAULT_RUBRIC).map( + (f) => (f.evidence?.result as { rule?: string })?.rule, + ) + // Line 8 (` , id`) in the second branch was added; the window on line 3 is old code. + expect(rules("@@ -7,1 +7,2 @@\n tenant\n+ , id\n")).not.toContain("dedup_no_tiebreaker") + expect(rules("@@ -3,1 +3,1 @@\n- , row_number() over () as rn\n+ , row_number() over (partition by tenant, id) as rn\n")).toContain( + "dedup_no_tiebreaker", + ) + }) }) describe("column changes without compiled SQL (dbt parse-only CI)", () => { @@ -350,4 +363,41 @@ describe("column changes without compiled SQL (dbt parse-only CI)", () => { }) expect(env.findings.find((f) => f.title.includes("on_schema_change"))?.body).toContain("adds `region`") }) + + test("columns inside a `{% if %}` block are unknown, not added", async () => { + const cfg = "{{ config(materialized='incremental', unique_key='id', on_schema_change='fail') }}\n" + const review = (oldSql: string, newSql: string) => + runReview({ + changedFiles: [{ path: "models/marts/fct.sql", status: "modified", diff: `-${oldSql}\n+${newSql}\n` }], + config: { ...DEFAULT_REVIEW_CONFIG, reviewers: ["dbt_patterns"] }, + rubric: DEFAULT_RUBRIC, + mode: "comment", + runner: { + async impact() { + return { hasManifest: true, severity: "SAFE", directCount: 0, transitiveCount: 0, testCount: 0 } + }, + async grade() { + return {} + }, + async check() { + return { issues: [], ran: true } + }, + async equivalence() { + return { decided: false } + }, + async detectPii() { + return { columns: [] } + }, + async columnLineage() { + return [] + }, + } as ReviewRunner, + getContent: async (_f, side) => (side === "old" ? oldSql : newSql), + }) + const conditional = await review( + `${cfg}select id, amount from {{ ref('src') }}`, + `${cfg}select id, amount {% if var('with_region', false) %}, region {% endif %} from {{ ref('src') }}`, + ) + expect(conditional.findings.some((f) => f.title.includes("on_schema_change"))).toBe(false) + }) }) diff --git a/packages/opencode/test/altimate/review-test-adequacy.test.ts b/packages/opencode/test/altimate/review-test-adequacy.test.ts index 644e664d14..49a4a7c9e7 100644 --- a/packages/opencode/test/altimate/review-test-adequacy.test.ts +++ b/packages/opencode/test/altimate/review-test-adequacy.test.ts @@ -31,6 +31,22 @@ describe("tests that compare a model with itself", () => { expect(detectTestAdequacy({ file, oldContent: yml("ref('fct_orders')"), newContent: yml("ref('fct_orders')") })).toEqual([]) }) + test("fewer_rows_than against itself, or an aggregation with a different expression, is not flagged", () => { + const y = `models: + - name: fct_orders + data_tests: + - dbt_utils.fewer_rows_than: + compare_model: ref('fct_orders') + - dbt_expectations.expect_table_aggregation_to_equal_other_table: + expression: sum(line_total) + compare_model: ref('fct_orders') + compare_expression: sum(order_total) +` + expect(detectTestAdequacy({ file, newContent: y })).toEqual([]) + const same = y.replace("sum(order_total)", "sum( line_total )") + expect(rule(detectTestAdequacy({ file, newContent: same }))).toEqual(["test-compares-model-with-itself"]) + }) + test("dbt 1.8+ `arguments:` form is read too", () => { const y = `models:\n - name: m\n data_tests:\n - dbt_utils.equality:\n arguments:\n compare_model: ref("m")\n` expect(rule(detectTestAdequacy({ file: { path: "models/_m.yml", status: "added" }, newContent: y }))).toEqual(["test-compares-model-with-itself"]) @@ -80,4 +96,15 @@ data_tests: expect(out.map((f) => f.title)).toEqual(["test thresholds loosened for `shop → marts`"]) expect(detectThresholdLoosening({ path: "dbt_project.yml", status: "modified" }, after, after)).toEqual([]) }) + + test("error_if: only a bound that tolerates more failures loosens; strict forms are the default", () => { + const at = (cond: string) => `name: shop\ndata_tests:\n shop:\n marts:\n +error_if: "${cond}"\n` + const loosened = (from: string, to: string) => + detectThresholdLoosening({ path: "dbt_project.yml", status: "modified" }, at(from), at(to)).length > 0 + expect(loosened(">100", ">10")).toBe(false) + expect(loosened(">10", ">100")).toBe(true) + expect(loosened("!= 0", "> 0")).toBe(false) + expect(loosened("!= 0", ">=1")).toBe(false) + expect(loosened("!= 0", ">5")).toBe(true) + }) })