diff --git a/docs/docs/usage/dbt-pr-review.md b/docs/docs/usage/dbt-pr-review.md index 1513e7c090..5d07e23204 100644 --- a/docs/docs/usage/dbt-pr-review.md +++ b/docs/docs/usage/dbt-pr-review.md @@ -209,13 +209,30 @@ jobs: | `target-base/compiled/…` | Compile the base ref into `target-base/` (below) | Equivalence proofs and column-breakage for modified models. | | `target/catalog.json` | `dbt docs generate` | Complete warehouse columns for column-lineage breakage. | -Compiling the base (optional, for equivalence and column-breakage): +The action can build these for you with `dbt_prepare` (off by default): + +```yaml + - uses: AltimateAI/altimate-code/github/review@main + with: + dbt_prepare: compile # none | compile | full + dbt_command: dbt # or "uv run dbt", "poetry run dbt" + dbt_profiles_dir: . # relative to project_dir +``` + +- **`compile`:** runs `dbt compile` on the head, then compiles the PR's merge-base + into `target-base/` in a temporary worktree. This enables equivalence and + column-breakage proofs. It usually works offline. +- **`full`:** also runs `dbt docs generate` for `catalog.json` (complete column + types). That needs warehouse credentials in the profile. +- A base compile that fails is a warning: the review still runs, and the + affected models are listed as not verified. + +To build them yourself, compile the **merge-base**, not the base branch's tip: ```yaml - name: Compile the base into target-base/ run: | - # The review diffs against the merge-base, not the target branch tip. - git worktree add ../base "$(git merge-base "origin/${{ github.base_ref }}" HEAD)" + git worktree add ../base "$(git merge-base origin/${{ github.base_ref }} HEAD)" (cd ../base && dbt deps && dbt compile --target-path "$GITHUB_WORKSPACE/target-base") ``` @@ -246,7 +263,7 @@ ways, or neither (it self-disables, leaving a deterministic-only review): Always pass keys as repo **secrets**. Warehouse credentials are consumed by the `dbt compile` step (via your `profiles.yml`), **not** by the review step. A complete, copy-paste workflow lives at -[`github/review/examples/altimate-ingestion.yml`](https://github.com/AltimateAI/altimate-code/blob/main/github/review/examples/altimate-ingestion.yml). +[`github/review/examples/dbt-pr-review.yml`](https://github.com/AltimateAI/altimate-code/blob/main/github/review/examples/dbt-pr-review.yml). Re-pushing commits updates the same summary comment in place; fixed findings are dropped on the next run. `--post` targets **GitHub** PRs (it reads `GITHUB_TOKEN` / @@ -500,6 +517,14 @@ The deterministic rules group into four families: - **dbt unit tests.** A unit test whose `given` inputs are all `rows: []` is reported: it either never tests the model's logic, or expects rows the model has nothing to build from. +- **Data tests that can't do their job.** These are checked on tests added or + changed in the PR: + - a test that compares a model with itself (`dbt_utils.equality` or + `equal_rowcount` against its own `ref`, or a `relationships` test pointing + at its own column) passes by construction; + - new tests that inherit `severity: warn` or an `error_if` threshold from + `dbt_project.yml` report failures without failing the build; + - a `dbt_project.yml` change that loosens those thresholds for a whole folder. - **Incremental and snapshot semantics.** Config is read from the manifest (project, yml and inline, as dbt merges them) with the head's inline `{{ config() }}` on top. Formatting-only edits never count as a change. diff --git a/github/review/action.yml b/github/review/action.yml index 09c4bc1538..02f6f7d7e9 100644 --- a/github/review/action.yml +++ b/github/review/action.yml @@ -31,6 +31,17 @@ inputs: description: "Post the verdict to the PR (summary comment + inline review)." required: false default: "true" + dbt_prepare: + description: "Build the dbt artifacts the full review uses before reviewing. `none` (default): you build them in earlier steps. `compile`: `dbt compile` on the head, and on the PR's merge-base into `target-base/` (equivalence and lineage proofs). `full`: also `dbt docs generate` for `catalog.json` (complete column types). `compile` usually works offline; `full` needs warehouse credentials in the profile." + required: false + default: "none" + dbt_command: + description: "How to invoke dbt for `dbt_prepare`, e.g. `dbt`, `uv run dbt`, `poetry run dbt`." + required: false + default: "dbt" + dbt_profiles_dir: + description: "`--profiles-dir` for `dbt_prepare`, relative to `project_dir`. Empty = dbt's default lookup." + required: false signing_key: description: "Optional HMAC key to sign the verdict envelope (ALTIMATE_REVIEW_SIGNING_KEY)." required: false @@ -167,6 +178,57 @@ runs: echo "Advisory lane: no model/credentials provided — running deterministic-only." fi + # Optional: build the head and base artifacts the full-tier review needs. + # Inputs are passed as ENV, never interpolated into the script. + - name: Prepare dbt artifacts + if: inputs.dbt_prepare != 'none' + shell: bash + env: + IN_PREPARE: ${{ inputs.dbt_prepare }} + IN_DBT: ${{ inputs.dbt_command }} + IN_PROFILES: ${{ inputs.dbt_profiles_dir }} + IN_PROJECT_DIR: ${{ inputs.project_dir }} + IN_BASE: ${{ inputs.base }} + EVENT_BASE: ${{ github.event.pull_request.base.ref }} + run: | + set -euo pipefail + case "$IN_PREPARE" in compile|full) ;; *) echo "::error::dbt_prepare must be none, compile or full (got '$IN_PREPARE')"; exit 1 ;; esac + read -ra DBT <<< "$IN_DBT" + PROJECT="$(cd "${IN_PROJECT_DIR:-.}" && pwd)" + REL="$(git -C "$PROJECT" rev-parse --show-prefix)" + PROFILES=() + [[ -n "${IN_PROFILES:-}" ]] && PROFILES=(--profiles-dir "$(cd "$PROJECT" && cd "$IN_PROFILES" && pwd)") + build() { # $1 = project dir, $2 = target path + ( cd "$1" + [[ -f packages.yml || -f dependencies.yml ]] && "${DBT[@]}" deps ${PROFILES[@]+"${PROFILES[@]}"} + "${DBT[@]}" compile --target-path "$2" ${PROFILES[@]+"${PROFILES[@]}"} ) + } + echo "Compiling the head…" + build "$PROJECT" "$PROJECT/target" + if [[ "$IN_PREPARE" == "full" ]]; then + echo "Generating catalog.json…" + ( cd "$PROJECT" && "${DBT[@]}" docs generate --target-path "$PROJECT/target" ${PROFILES[@]+"${PROFILES[@]}"} ) || + echo "::warning::dbt docs generate failed; continuing without catalog.json." + fi + # The base side: the PR's merge-base, compiled into target-base/. + BASE="${IN_BASE:-$EVENT_BASE}" + BASE="${BASE#origin/}" + if [[ -z "$BASE" ]]; then echo "::warning::No base branch known; skipping the base compile."; exit 0; fi + git fetch --no-tags origin "+refs/heads/$BASE:refs/remotes/origin/$BASE" >/dev/null 2>&1 || true + MB="$(git merge-base "origin/$BASE" HEAD 2>/dev/null || git merge-base "$BASE" HEAD 2>/dev/null || true)" + if [[ -z "$MB" && "$(git rev-parse --is-shallow-repository)" == "true" ]]; then + git fetch --no-tags --deepen=1000 origin "+refs/heads/$BASE:refs/remotes/origin/$BASE" "$(git rev-parse HEAD)" >/dev/null 2>&1 || true + MB="$(git merge-base "origin/$BASE" HEAD 2>/dev/null || true)" + fi + if [[ -z "$MB" ]]; then echo "::warning::Could not find the merge-base with $BASE; skipping the base compile."; exit 0; fi + WT="$RUNNER_TEMP/altimate-review-base" + trap 'git worktree remove --force "$WT" >/dev/null 2>&1 || true' EXIT + git worktree add --detach "$WT" "$MB" >/dev/null || { echo "::warning::Could not check out the base; skipping the base compile."; exit 0; } + # A stale target-base/ from an earlier run must not stand in for a failed compile. + rm -rf "$PROJECT/target-base" + echo "Compiling the base ($BASE @ ${MB:0:12})…" + build "$WT/$REL" "$PROJECT/target-base" || echo "::warning::The base compile failed; equivalence will be marked unverified." + - name: Run dbt PR review shell: bash # Inputs are passed as ENV (data), never interpolated into the command — diff --git a/github/review/examples/altimate-ingestion.yml b/github/review/examples/dbt-pr-review.yml similarity index 97% rename from github/review/examples/altimate-ingestion.yml rename to github/review/examples/dbt-pr-review.yml index dc52867909..799d424424 100644 --- a/github/review/examples/altimate-ingestion.yml +++ b/github/review/examples/dbt-pr-review.yml @@ -1,4 +1,4 @@ -# Example: drop-in dbt PR review for a dbt repo (e.g. altimate-ingestion). +# Example: drop-in dbt PR review for a dbt repo. # Copy to the target repo as `.github/workflows/dbt-pr-review.yml` and adjust the # dbt adapter + profiles/secrets to match the warehouse. # diff --git a/packages/opencode/src/altimate/review/dbt-patterns.ts b/packages/opencode/src/altimate/review/dbt-patterns.ts index 91053d7515..02ceb357f5 100644 --- a/packages/opencode/src/altimate/review/dbt-patterns.ts +++ b/packages/opencode/src/altimate/review/dbt-patterns.ts @@ -225,15 +225,70 @@ const detectNotIn: Detector = (c) => { }) } +/** Head-side line numbers of the added lines; undefined when the diff has no hunk headers. */ +function addedLineNumbers(diff: string | undefined): Set | undefined { + if (!diff || !/^@@ /m.test(diff)) return undefined + const out = new Set() + let right = 0 + let inHunk = false + for (const line of diff.split("\n")) { + const h = /^@@ -\d+(?:,\d+)? \+(\d+)(?:,\d+)? @@/.exec(line) + if (h) { + right = Number(h[1]) + inHunk = true + continue + } + if (!inHunk || line.startsWith("\\")) continue + if (line.startsWith("+")) out.add(right++) + else if (line.startsWith(" ") || line === "") right++ + else if (!line.startsWith("-")) inHunk = false + } + return out +} + +/** + * `row_number() over (…)` windows in the model, read from the whole file so a + * window split across lines is seen whole. A window counts as part of the + * change when one of the PR's added lines falls inside its line span (by text + * only when the diff carries no hunk headers). + */ +function dedupWindows(c: Ctx): Array<{ spec: string; text: string }> { + // Comments blanked with newlines kept, so offsets map to head line numbers. + const sql = c.newSql.replace(/\/\*[\s\S]*?\*\/|--[^\n]*|\{#[\s\S]*?#\}/g, (m) => m.replace(/[^\n]/g, " ")) + const lineAt = (offset: number) => sql.slice(0, offset).split("\n").length + const addedLines = addedLineNumbers(c.file.diff) + const added = c.added.map((l) => stripComments(l).trim()).filter((l) => l.length >= 3) + const out: Array<{ spec: string; text: string }> = [] + for (const m of sql.matchAll(/row_number\s*\(\s*\)\s*over\s*\(/gi)) { + const open = m.index! + m[0].length - 1 + let depth = 0 + let close = -1 + for (let i = open; i < sql.length; i++) { + if (sql[i] === "(") depth++ + else if (sql[i] === ")" && --depth === 0) { + close = i + break + } + } + if (close < 0) continue + const text = sql.slice(m.index!, close + 1) + const touched = addedLines + ? [...addedLines].some((n) => n >= lineAt(m.index!) && n <= lineAt(close)) + : added.some((l) => text.includes(l) || l.includes(text.replace(/\s+/g, " "))) + if (!touched) continue + out.push({ spec: sql.slice(open + 1, close).replace(/\s+/g, " ").trim(), text: text.replace(/\s+/g, " ") }) + } + return out +} + // 9. Dedup via ROW_NUMBER()/QUALIFY whose ORDER BY has no unique tiebreaker. const detectDedupTie: Detector = (c) => { - const hit = c.added.find((l) => /row_number\s*\(\s*\)\s*over\s*\(/i.test(stripComments(l))) - if (!hit) return null // Only flag dedup with NO ORDER BY — that is unambiguously non-deterministic. // A present ORDER BY (even single-column) is the developer's deterministic // choice and the dbt-recommended pattern; warning "it might not be unique" // on every dedup is noise (false positives on the common, correct case). - if (/order\s+by/i.test(stripComments(hit))) return null + const hit = dedupWindows(c).find((w) => !/order\s+by/i.test(w.spec)) + if (!hit) return null return makeFinding({ severity: "warning", category: "dedup", @@ -244,7 +299,7 @@ const detectDedupTie: Detector = (c) => { file: c.file.path, model: c.model, confidence: "high", - evidence: { tool: "dbt-patterns", result: { rule: "dedup_no_tiebreaker", line: hit.trim() } }, + evidence: { tool: "dbt-patterns", result: { rule: "dedup_no_tiebreaker", line: hit.text } }, ruleKey: "dedup:no-tiebreaker", }) } @@ -257,11 +312,8 @@ const detectDedupTie: Detector = (c) => { const LOAD_TS_RE = /^(_?(ingest(ion|ed)?|load(ed)?|etl|extract(ed)?|batch)(_at|_ts|_time|_timestamp|_date|_dt)|_?(ingestion|load|etl|batch)_timestamp|_loaded_at)$/i const detectDedupLoadTimestamp: Detector = (c) => { - for (const line of c.added) { - const s = stripComments(line) - const m = /row_number\s*\(\s*\)\s*over\s*\(([^)]*)\)/i.exec(s) - if (!m) continue - const order = /order\s+by\s+(.+)$/i.exec(m[1])?.[1] + for (const w of dedupWindows(c)) { + const order = /order\s+by\s+(.+)$/i.exec(w.spec)?.[1] if (!order) continue const keys = order.split(",").map((k) => k.trim().replace(/\s+(asc|desc)\b.*$/i, "").replace(/\s+nulls\s+(first|last)$/i, "")) if (keys.length !== 1) continue @@ -277,7 +329,7 @@ const detectDedupLoadTimestamp: Detector = (c) => { file: c.file.path, model: c.model, confidence: "high", - evidence: { tool: "dbt-patterns", result: { rule: "dedup_load_timestamp_only", line: line.trim() } }, + evidence: { tool: "dbt-patterns", result: { rule: "dedup_load_timestamp_only", line: w.text } }, ruleKey: "dedup:load-timestamp-only", }) } diff --git a/packages/opencode/src/altimate/review/orchestrate.ts b/packages/opencode/src/altimate/review/orchestrate.ts index 701118b058..d1426b4a35 100644 --- a/packages/opencode/src/altimate/review/orchestrate.ts +++ b/packages/opencode/src/altimate/review/orchestrate.ts @@ -26,6 +26,7 @@ import { detectModelPatterns, detectSchemaYmlPatterns, splitDiff } from "./dbt-p import { listMirrorsModel, type ConsumerColumnList } from "./consumer-lists" import { detectUnitTestFixtures } from "./unit-tests" import { selectListNames } from "./select-list" +import { detectTestAdequacy, detectThresholdLoosening } from "./test-adequacy" import { incrementalFindings, snapshotFindings, snapshotYmlFindings } from "./incremental-semantics" import { type AiLaneStatus, type AiReviewInput, type AiReviewResult, safeReason } from "./ai-review" @@ -661,19 +662,44 @@ async function columnDelta( dialect: string, ): Promise<{ added: string[]; removed: string[]; current: Set }> { const none = { added: [], removed: [], current: new Set() } - const { file, engineNewSql, engineOldSql } = ctx - if (!engineNewSql || file.status === "deleted") return none - const current = await outputColumns(runner, engineNewSql, dialect) - if (!current.size) return none - if (!hasOldSide(file)) return { added: [...current], removed: [], current } - if (!engineOldSql) return none - const before = await outputColumns(runner, engineOldSql, dialect) - if (!before.size) return none - return { - added: [...current].filter((c) => !before.has(c)), - removed: [...before].filter((c) => !current.has(c)), - current, + const { file, engineNewSql, engineOldSql, newSql, oldSql } = ctx + if (file.status === "deleted") return none + const withOld = hasOldSide(file) + const delta = (current: Set, before?: Set) => + !current.size || (withOld && !before?.size) + ? none + : { + added: [...current].filter((c) => !before?.has(c)), + removed: before ? [...before].filter((c) => !current.has(c)) : [], + current, + } + // Compiled SQL on both sides: column lineage plus the final SELECT's names. + if (engineNewSql && (!withOld || engineOldSql)) { + const current = await outputColumns(runner, engineNewSql, dialect) + const before = withOld ? await outputColumns(runner, engineOldSql!, dialect) : undefined + return delta(current, before) } + // No compiled SQL (CI that only runs `dbt parse`): the raw model's own SELECT + // list, when it names its columns. Columns produced inside a macro stay unknown. + if (!newSql) return none + const current = rawSelectList(newSql) + const before = withOld && oldSql ? rawSelectList(oldSql) : undefined + return delta(current, before) +} + +/** Output names of a raw (Jinja) model's final SELECT; empty when a Jinja + * expression or tag is in the list. */ +function rawSelectList(sql: string): Set { + const plain = sql + .replace(/\{#[\s\S]*?#\}/g, " ") + // A tag in the list (`{% if %}` / `{% for %}`) makes its columns conditional: + // it becomes its own `__jinja__` item, which marks the list unknown. + .replace(/\{%[\s\S]*?%\}/g, " , __jinja__ , ") + .replace(/\{\{[\s\S]*?\}\}/g, " __jinja__ ") + const names = selectListNames(plain) + if (!names || names.includes("__jinja__")) return new Set() + // A Jinja expression inside an item (e.g. `{{ col }} as x`) leaves its alias intact. + return new Set(names) } const ticks = (xs: string[]) => xs.map((x) => `\`${x}\``).join(", ") @@ -1725,6 +1751,9 @@ export async function runReview(input: OrchestrateInput): Promise f.kind === "schema_yml") + // The head's dbt_project.yml (test thresholds inherited by new tests), read once. + const projectYmlPath = `${input.layout?.root ? `${input.layout.root.replace(/\/$/, "")}/` : ""}dbt_project.yml` + const projectYml = schemaFiles.length ? await getContent?.(projectYmlPath, "new").catch(() => undefined) : undefined if (schemaFiles.length) { const schemaFindingSets = await Promise.all( schemaFiles.map(async (file) => { @@ -1738,6 +1767,7 @@ export async function runReview(input: OrchestrateInput): Promise f.kind === "project_config" && /(^|\/)dbt_project\.ya?ml$/.test(f.path))) { + if (file.status !== "modified") continue + const [oldContent, newContent] = await Promise.all([getContent?.(file.path, "old"), getContent?.(file.path, "new")]) + all.push(detectThresholdLoosening(file, oldContent, newContent)) + } // Architectural dedup: the regex `dbt-patterns`/`rule-catalog` layer is a // diff-aware, Jinja-aware, offline fallback. For the STRUCTURAL SQL checks diff --git a/packages/opencode/src/altimate/review/test-adequacy.ts b/packages/opencode/src/altimate/review/test-adequacy.ts new file mode 100644 index 0000000000..7d2713719e --- /dev/null +++ b/packages/opencode/src/altimate/review/test-adequacy.ts @@ -0,0 +1,238 @@ +import YAML from "yaml" +import { makeFinding, type Finding } from "./finding" +import type { ChangedFile, DbtProjectLayout } from "./diff-filter" + +/** + * Data tests that cannot do their job, found from yml alone (any adapter): + * - a test that compares a model with itself (`dbt_utils.equality` / + * `equal_rowcount` against its own `ref`, or a `relationships` test pointing + * at its own column) passes by construction; + * - tests added in the PR that inherit `severity: warn` (or an `error_if` + * threshold) from `dbt_project.yml` report failures without failing the build; + * - a PR that loosens those thresholds in `dbt_project.yml` weakens every test + * under that path. + * Only tests added or changed in the PR are checked. + */ + +interface TestRef { + entity: string + column?: string + test: string + args: Record +} + +const SELF_COMPARE = new Set([ + "equality", + "equal_rowcount", + "cardinality_equality", + "expect_table_row_count_to_equal_other_table", + "expect_table_aggregation_to_equal_other_table", +]) + +function parse(text?: string): any { + if (!text) return undefined + try { + return YAML.parse(text) + } catch { + return undefined + } +} + +function testsOf(doc: any): TestRef[] { + const out: TestRef[] = [] + const read = (list: unknown, entity: string, column?: string) => { + if (!Array.isArray(list)) return + for (const t of list) { + if (typeof t === "string") out.push({ entity, column, test: t, args: {} }) + else if (t && typeof t === "object") { + const [k] = Object.keys(t) + if (k) out.push({ entity, column, test: k, args: (t as any)[k] ?? {} }) + } + } + } + for (const group of ["models", "snapshots", "seeds"]) { + for (const e of Array.isArray(doc?.[group]) ? doc[group] : []) { + if (typeof e?.name !== "string") continue + read(e.data_tests ?? e.tests, e.name) + for (const c of Array.isArray(e.columns) ? e.columns : []) + if (typeof c?.name === "string") read(c.data_tests ?? c.tests, e.name, c.name) + } + } + return out +} + +const key = (t: TestRef) => `${t.entity}|${t.column ?? ""}|${t.test}|${JSON.stringify(t.args)}` +const arg = (t: TestRef, name: string) => t.args?.[name] ?? t.args?.arguments?.[name] +const shortName = (test: string) => test.split(".").pop()!.toLowerCase() +const refsEntity = (v: unknown, entity: string) => + typeof v === "string" && new RegExp(`ref\\(\\s*['"]${entity.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")}['"]\\s*\\)`).test(v) + +function finding(file: ChangedFile, model: string | undefined, rule: string, severity: Finding["severity"], title: string, body: string, extra = ""): Finding { + return makeFinding({ + severity, + category: "test_coverage", + title, + body, + file: file.path, + model, + confidence: "high", + evidence: { tool: "test-adequacy", result: { rule } }, + ruleKey: `test_coverage:${rule}${extra}`, + }) +} + +/** + * Failing rows an `error_if` condition tolerates: `!= 0` / `> 0` → 0, `> 10` → 10, + * `>= 10` → 9. Undefined when the condition isn't a simple bound. + */ +function allowedFailures(cond: string | undefined): number | undefined { + const m = cond ? /^(!=|<>|>=|>)\s*(\d+)$/.exec(cond.trim()) : null + if (!m) return undefined + const n = Number(m[2]) + if (m[1] === ">=") return Math.max(0, n - 1) + if (m[1] === ">") return n + return n === 0 ? 0 : undefined +} + +/** `+severity` / `+error_if` / `+warn_if` (with or without `+`) on one config node. */ +function thresholds(node: any): { severity?: string; error_if?: string; warn_if?: string } { + if (!node || typeof node !== "object") return {} + const get = (k: string) => (node[`+${k}`] ?? node[k]) as unknown + const str = (v: unknown) => (typeof v === "string" ? v.trim().toLowerCase() : undefined) + return { severity: str(get("severity")), error_if: str(get("error_if")), warn_if: str(get("warn_if")) } +} + +/** + * Effective test thresholds for a yml under `segments` (folders below the model + * path), walking `data_tests:` (or `tests:`) from the top, through the project + * name, down the folders; deeper levels override. `path` is where the effective + * value was set. + */ +export function inheritedThresholds(projectYml: any, segments: string[]): { severity?: string; error_if?: string; path: string[] } { + const tree = projectYml?.data_tests ?? projectYml?.tests + if (!tree || typeof tree !== "object") return { path: [] } + const name = projectYml?.name + const chain = name && tree[name] && typeof tree[name] === "object" ? [String(name), ...segments] : segments + const levels: Array<[string[], any]> = [[[], tree]] + let node = tree + for (let i = 0; i < chain.length; i++) { + node = node?.[chain[i]] + if (!node || typeof node !== "object") break + levels.push([chain.slice(0, i + 1), node]) + } + const eff: { severity?: string; error_if?: string } = {} + let path: string[] = [] + for (const [at, n] of levels) { + const t = thresholds(n) + if (t.severity) eff.severity = t.severity + if (t.error_if) eff.error_if = t.error_if + if (t.severity || t.error_if) path = at.length ? at : ["(all tests)"] + } + return { ...eff, path } +} + +/** Folders of a yml below its model path (e.g. `models/marts/x.yml` → `["marts"]`). */ +function segmentsOf(file: string, layout?: DbtProjectLayout): string[] { + const root = layout?.root ? layout.root.replace(/\/$/, "") + "/" : "" + const rel = root && file.startsWith(root) ? file.slice(root.length) : file + const parts = rel.split("/").slice(0, -1) + const modelDirs = layout?.dirs.model ?? ["models"] + return modelDirs.includes(parts[0]) ? parts.slice(1) : parts +} + +export function detectTestAdequacy(input: { + file: ChangedFile + oldContent?: string + newContent?: string + projectYml?: string + layout?: DbtProjectLayout +}): Finding[] { + const { file } = input + if (file.status === "deleted") return [] + const newDoc = parse(input.newContent) + if (!newDoc) return [] + const before = new Set(testsOf(parse(input.oldContent)).map(key)) + const added = testsOf(newDoc).filter((t) => !before.has(key(t))) + const out: Finding[] = [] + + for (const t of added) { + const name = shortName(t.test) + // Different expressions on the same model are a valid invariant (e.g. sum of + // line items equals the order total); only the same expression is circular. + const sameExpression = + name !== "expect_table_aggregation_to_equal_other_table" || + String(arg(t, "compare_expression") ?? arg(t, "expression") ?? "").replace(/\s+/g, "").toLowerCase() === + String(arg(t, "expression") ?? "").replace(/\s+/g, "").toLowerCase() + const self = + (SELF_COMPARE.has(name) && sameExpression && ["compare_model", "to", "compare_to"].some((a) => refsEntity(arg(t, a), t.entity))) || + (name === "relationships" && refsEntity(arg(t, "to"), t.entity) && String(arg(t, "field") ?? "").toLowerCase() === (t.column ?? "").toLowerCase()) + if (self) + out.push( + finding(file, t.entity, "test-compares-model-with-itself", "warning", `${t.entity}: \`${t.test}\` compares the model with itself`, + `This test checks \`${t.entity}\`${t.column ? `.\`${t.column}\`` : ""} against \`ref('${t.entity}')\`, so it passes by construction. ` + + "Point it at the independent source of truth it is meant to reconcile against.", `:${key(t)}`), + ) + } + + const project = parse(input.projectYml) + if (project && added.length) { + const inherited = inheritedThresholds(project, segmentsOf(file.path, input.layout)) + const own = (t: TestRef) => { + const c = t.args?.config ?? {} + return [c.severity, c.error_if, c.warn_if, t.args?.severity, t.args?.error_if].some((v) => v !== undefined) + } + const loose = added.filter((t) => !own(t)) + const where = inherited.path.length ? ` (\`data_tests:\` → ${inherited.path.map((p) => `\`${p}\``).join(" → ")})` : "" + if (loose.length && inherited.severity === "warn") + out.push( + finding(file, undefined, "test-inherits-warn", "suggestion", `${loose.length} new test(s) in ${file.path} inherit \`severity: warn\``, + `They inherit \`severity: warn\` from \`dbt_project.yml\`${where}, so failures are reported but never fail \`dbt build\`. ` + + "If a test should gate the build, set `config: {severity: error}` on it."), + ) + else if (loose.length && inherited.error_if && allowedFailures(inherited.error_if) !== 0) + out.push( + finding(file, undefined, "test-inherits-threshold", "suggestion", `${loose.length} new test(s) in ${file.path} inherit \`error_if: ${inherited.error_if}\``, + `They fail only when \`${inherited.error_if}\` rows fail, a threshold inherited from \`dbt_project.yml\`${where}. ` + + "Set `config: {error_if: '!= 0'}` on tests that must catch any bad row."), + ) + } + return out +} + +/** A `dbt_project.yml` change that loosens test thresholds for a whole path. */ +export function detectThresholdLoosening(file: ChangedFile, oldContent?: string, newContent?: string): Finding[] { + const oldDoc = parse(oldContent) + const newDoc = parse(newContent) + if (!oldDoc || !newDoc) return [] + const flat = (doc: any) => { + const out = new Map>() + const visit = (node: any, path: string[]) => { + if (!node || typeof node !== "object") return + out.set(path.join(" → ") || "(all tests)", thresholds(node)) + for (const [k, v] of Object.entries(node)) if (!k.startsWith("+") && v && typeof v === "object") visit(v, [...path, k]) + } + visit(doc?.data_tests ?? doc?.tests, []) + return out + } + const before = flat(oldDoc) + const out: Finding[] = [] + for (const [path, t] of flat(newDoc)) { + const prev = before.get(path) ?? {} + const toWarn = t.severity === "warn" && prev.severity !== "warn" + // Only a threshold that tolerates more failing rows than before loosens; + // a changed condition that is not a simple bound is still reported. + const allowed = allowedFailures(t.error_if) + const newThreshold = + !!t.error_if && + t.error_if !== prev.error_if && + (allowed === undefined || allowed > (allowedFailures(prev.error_if) ?? 0)) + if (!toWarn && !newThreshold) continue + out.push( + finding(file, undefined, "test-thresholds-loosened", "warning", `test thresholds loosened for \`${path}\``, + (toWarn ? "`severity: warn` now applies" : `\`error_if: ${t.error_if}\` now applies`) + + ` to every data test under \`${path}\` in \`dbt_project.yml\`, so those tests stop failing the build. ` + + "Prefer setting it on the specific tests that need it.", `:${path}`), + ) + } + return out +} diff --git a/packages/opencode/test/altimate/review-action.test.ts b/packages/opencode/test/altimate/review-action.test.ts new file mode 100644 index 0000000000..fda2d21a26 --- /dev/null +++ b/packages/opencode/test/altimate/review-action.test.ts @@ -0,0 +1,33 @@ +import { describe, test, expect } from "bun:test" +import { promises as fs } from "node:fs" +import path from "node:path" +import YAML from "yaml" + +/** The GitHub Action's optional artifact preparation stays opt-in and injection-safe. */ +const actionPath = path.join(import.meta.dir, "../../../../github/review/action.yml") + +describe("action: dbt_prepare", () => { + test("is opt-in, and its script reads inputs only from env", async () => { + const action = YAML.parse(await fs.readFile(actionPath, "utf8")) + expect(action.inputs.dbt_prepare.default).toBe("none") + expect(action.inputs.dbt_command.default).toBe("dbt") + const steps: Array<{ name: string; if?: string; run?: string; env?: Record }> = action.runs.steps + const prep = steps.find((s) => s.name === "Prepare dbt artifacts")! + const review = steps.findIndex((s) => s.name === "Run dbt PR review") + expect(steps.indexOf(prep)).toBeLessThan(review) + expect(prep.if).toBe("inputs.dbt_prepare != 'none'") + expect(prep.run).not.toContain("${{") + expect(prep.env?.IN_DBT).toBe("${{ inputs.dbt_command }}") + // The base is compiled into target-base/, where the compiled-SQL resolver looks. + expect(prep.run).toContain('"$PROJECT/target-base"') + }) + + test("a failed docs generate or base checkout warns and lets the review run", async () => { + const action = YAML.parse(await fs.readFile(actionPath, "utf8")) + const prep = (action.runs.steps as Array<{ name: string; run?: string }>).find((s) => s.name === "Prepare dbt artifacts")! + const run = prep.run! + expect(run).toMatch(/docs generate[^\n]*\) \|\|\n\s*echo "::warning::/) + expect(run).toMatch(/git worktree add[^\n]*\|\| \{ echo "::warning::/) + expect(run.indexOf('rm -rf "$PROJECT/target-base"')).toBeLessThan(run.indexOf('build "$WT/$REL"')) + }) +}) diff --git a/packages/opencode/test/altimate/review-incremental-semantics.test.ts b/packages/opencode/test/altimate/review-incremental-semantics.test.ts index bbf424ceb6..ffddb310b7 100644 --- a/packages/opencode/test/altimate/review-incremental-semantics.test.ts +++ b/packages/opencode/test/altimate/review-incremental-semantics.test.ts @@ -299,3 +299,105 @@ describe("end to end through the orchestrator", () => { expect(env.verdict).not.toBe("REQUEST_CHANGES") }) }) + +describe("dedup windows split across lines", () => { + const sql = "select *,\n row_number() over (\n partition by job_id\n order by ingestion_timestamp desc\n ) as rn\nfrom t" + const rulesFor = (diffAdded: string) => + detectModelPatterns({ path: "models/m.sql", status: "modified", diff: diffAdded }, sql, DEFAULT_RUBRIC).map( + (f) => (f.evidence?.result as { rule?: string })?.rule, + ) + test("an ORDER BY on its own line is seen: no false 'no ORDER BY', and the load-timestamp rule fires", () => { + const r = rulesFor("+ row_number() over (\n") + expect(r).not.toContain("dedup_no_tiebreaker") + expect(r).toContain("dedup_load_timestamp_only") + }) + test("a window the PR doesn't touch is not reported", () => { + expect(rulesFor("+select 1 as unrelated\n")).not.toContain("dedup_load_timestamp_only") + }) + test("with hunk headers, an added line matching window text elsewhere doesn't touch the window", () => { + const noOrder = + "select\n tenant\n , row_number() over (partition by tenant, id) as rn\nfrom t\nunion all\nselect\n tenant\n , id\nfrom u" + const rules = (diff: string) => + detectModelPatterns({ path: "models/m.sql", status: "modified", diff }, noOrder, DEFAULT_RUBRIC).map( + (f) => (f.evidence?.result as { rule?: string })?.rule, + ) + // Line 8 (` , id`) in the second branch was added; the window on line 3 is old code. + expect(rules("@@ -7,1 +7,2 @@\n tenant\n+ , id\n")).not.toContain("dedup_no_tiebreaker") + expect(rules("@@ -3,1 +3,1 @@\n- , row_number() over () as rn\n+ , row_number() over (partition by tenant, id) as rn\n")).toContain( + "dedup_no_tiebreaker", + ) + }) +}) + +describe("column changes without compiled SQL (dbt parse-only CI)", () => { + test("on_schema_change='fail' sees columns added to the raw model's SELECT", async () => { + const cfg = "{{ config(materialized='incremental', unique_key='id', on_schema_change='fail') }}\n" + const oldSql = `${cfg}select id, amount from {{ ref('src') }}` + const newSql = `${cfg}select id, amount, region from {{ ref('src') }}` + const env = await runReview({ + changedFiles: [{ path: "models/marts/fct.sql", status: "modified", diff: `-${oldSql}\n+${newSql}\n` }], + config: { ...DEFAULT_REVIEW_CONFIG, reviewers: ["dbt_patterns"] }, + rubric: DEFAULT_RUBRIC, + mode: "comment", + runner: { + async impact() { + return { hasManifest: true, severity: "SAFE", directCount: 0, transitiveCount: 0, testCount: 0 } + }, + async grade() { + return {} + }, + async check() { + return { issues: [], ran: true } + }, + async equivalence() { + return { decided: false } + }, + async detectPii() { + return { columns: [] } + }, + async columnLineage() { + return [] + }, + } as ReviewRunner, + getContent: async (_f, side) => (side === "old" ? oldSql : newSql), + }) + expect(env.findings.find((f) => f.title.includes("on_schema_change"))?.body).toContain("adds `region`") + }) + + test("columns inside a `{% if %}` block are unknown, not added", async () => { + const cfg = "{{ config(materialized='incremental', unique_key='id', on_schema_change='fail') }}\n" + const review = (oldSql: string, newSql: string) => + runReview({ + changedFiles: [{ path: "models/marts/fct.sql", status: "modified", diff: `-${oldSql}\n+${newSql}\n` }], + config: { ...DEFAULT_REVIEW_CONFIG, reviewers: ["dbt_patterns"] }, + rubric: DEFAULT_RUBRIC, + mode: "comment", + runner: { + async impact() { + return { hasManifest: true, severity: "SAFE", directCount: 0, transitiveCount: 0, testCount: 0 } + }, + async grade() { + return {} + }, + async check() { + return { issues: [], ran: true } + }, + async equivalence() { + return { decided: false } + }, + async detectPii() { + return { columns: [] } + }, + async columnLineage() { + return [] + }, + } as ReviewRunner, + getContent: async (_f, side) => (side === "old" ? oldSql : newSql), + }) + const conditional = await review( + `${cfg}select id, amount from {{ ref('src') }}`, + `${cfg}select id, amount {% if var('with_region', false) %}, region {% endif %} from {{ ref('src') }}`, + ) + expect(conditional.findings.some((f) => f.title.includes("on_schema_change"))).toBe(false) + }) +}) diff --git a/packages/opencode/test/altimate/review-test-adequacy.test.ts b/packages/opencode/test/altimate/review-test-adequacy.test.ts new file mode 100644 index 0000000000..49a4a7c9e7 --- /dev/null +++ b/packages/opencode/test/altimate/review-test-adequacy.test.ts @@ -0,0 +1,110 @@ +import { describe, test, expect } from "bun:test" +import YAML from "yaml" +import { detectTestAdequacy, detectThresholdLoosening, inheritedThresholds } from "../../src/altimate/review/test-adequacy" + +/** Data tests that can't do their job, from yml alone (any adapter). */ + +const file = { path: "models/marts/_fct.yml", status: "modified" as const } +const rule = (fs: { evidence?: { result?: unknown } }[]) => fs.map((f) => (f.evidence?.result as { rule: string }).rule) + +describe("tests that compare a model with itself", () => { + const yml = (compare: string) => `models: + - name: fct_orders + data_tests: + - dbt_utils.equal_rowcount: + compare_model: ${compare} + columns: + - name: customer_id + data_tests: + - relationships: + to: ref('fct_orders') + field: customer_id +` + test("own-ref comparisons and self relationships pass by construction", () => { + const out = detectTestAdequacy({ file, newContent: yml("ref('fct_orders')") }) + expect(rule(out)).toEqual(["test-compares-model-with-itself", "test-compares-model-with-itself"]) + }) + + test("a comparison with another model is fine; unchanged tests are skipped", () => { + const other = yml("ref('stg_orders')").replace("to: ref('fct_orders')", "to: ref('dim_customers')") + expect(detectTestAdequacy({ file, newContent: other })).toEqual([]) + expect(detectTestAdequacy({ file, oldContent: yml("ref('fct_orders')"), newContent: yml("ref('fct_orders')") })).toEqual([]) + }) + + test("fewer_rows_than against itself, or an aggregation with a different expression, is not flagged", () => { + const y = `models: + - name: fct_orders + data_tests: + - dbt_utils.fewer_rows_than: + compare_model: ref('fct_orders') + - dbt_expectations.expect_table_aggregation_to_equal_other_table: + expression: sum(line_total) + compare_model: ref('fct_orders') + compare_expression: sum(order_total) +` + expect(detectTestAdequacy({ file, newContent: y })).toEqual([]) + const same = y.replace("sum(order_total)", "sum( line_total )") + expect(rule(detectTestAdequacy({ file, newContent: same }))).toEqual(["test-compares-model-with-itself"]) + }) + + test("dbt 1.8+ `arguments:` form is read too", () => { + const y = `models:\n - name: m\n data_tests:\n - dbt_utils.equality:\n arguments:\n compare_model: ref("m")\n` + expect(rule(detectTestAdequacy({ file: { path: "models/_m.yml", status: "added" }, newContent: y }))).toEqual(["test-compares-model-with-itself"]) + }) +}) + +describe("thresholds inherited from dbt_project.yml", () => { + const project = `name: shop +data_tests: + shop: + marts: + +severity: warn + staging: + +error_if: ">100" +` + const newTests = (own = "") => `models: + - name: fct_orders + columns: + - name: id + data_tests: + - not_null${own} +` + test("the effective value walks project → folders, deeper wins", () => { + const doc = YAML.parse(project) + expect(inheritedThresholds(doc, ["marts"])).toEqual({ severity: "warn", path: ["shop", "marts"] }) + expect(inheritedThresholds(doc, ["staging", "x"])).toEqual({ error_if: ">100", path: ["shop", "staging"] }) + expect(inheritedThresholds(doc, ["core"])).toEqual({ path: [] }) + }) + + test("new tests under a warn folder are reported; a test with its own severity is not", () => { + const out = detectTestAdequacy({ file, newContent: newTests(), projectYml: project }) + expect(rule(out)).toEqual(["test-inherits-warn"]) + expect(out[0].body).toContain("`shop` → `marts`") + const own = newTests(":\n config:\n severity: error").replace("- not_null:", "- not_null:") + expect(detectTestAdequacy({ file, newContent: own, projectYml: project })).toEqual([]) + }) + + test("an inherited error_if threshold is reported", () => { + const out = detectTestAdequacy({ file: { path: "models/staging/_stg.yml", status: "modified" }, newContent: newTests(), projectYml: project }) + expect(rule(out)).toEqual(["test-inherits-threshold"]) + }) + + test("a dbt_project.yml change that loosens thresholds is reported", () => { + const before = "name: shop\ndata_tests:\n shop:\n marts:\n +severity: error\n" + const after = "name: shop\ndata_tests:\n shop:\n marts:\n +severity: warn\n" + const out = detectThresholdLoosening({ path: "dbt_project.yml", status: "modified" }, before, after) + expect(out.map((f) => f.title)).toEqual(["test thresholds loosened for `shop → marts`"]) + expect(detectThresholdLoosening({ path: "dbt_project.yml", status: "modified" }, after, after)).toEqual([]) + }) + + test("error_if: only a bound that tolerates more failures loosens; strict forms are the default", () => { + const at = (cond: string) => `name: shop\ndata_tests:\n shop:\n marts:\n +error_if: "${cond}"\n` + const loosened = (from: string, to: string) => + detectThresholdLoosening({ path: "dbt_project.yml", status: "modified" }, at(from), at(to)).length > 0 + expect(loosened(">100", ">10")).toBe(false) + expect(loosened(">10", ">100")).toBe(true) + expect(loosened("!= 0", "> 0")).toBe(false) + expect(loosened("!= 0", ">=1")).toBe(false) + expect(loosened("!= 0", ">5")).toBe(true) + }) +}) diff --git a/packages/opencode/test/skill/release-v0.8.5-adversarial.test.ts b/packages/opencode/test/skill/release-v0.8.5-adversarial.test.ts index 00b1ee1724..77d91cbf04 100644 --- a/packages/opencode/test/skill/release-v0.8.5-adversarial.test.ts +++ b/packages/opencode/test/skill/release-v0.8.5-adversarial.test.ts @@ -238,7 +238,7 @@ describe("v0.8.5 adversarial - composite action", () => { const changelog = await fs.readFile(path.join(repoRoot, "CHANGELOG.md"), "utf8") expect(changelog).toContain(`## [${version}]`) - for (const relative of ["docs/docs/usage/dbt-pr-review.md", "github/review/examples/altimate-ingestion.yml"]) { + for (const relative of ["docs/docs/usage/dbt-pr-review.md", "github/review/examples/dbt-pr-review.yml"]) { const content = await fs.readFile(path.join(repoRoot, relative), "utf8") expect(content).toContain(`AltimateAI/altimate-code/github/review@v${version}`) expect(content).not.toContain("AltimateAI/altimate-code/github/review@v0.8.4")