diff --git a/docs/docs/usage/dbt-pr-review.md b/docs/docs/usage/dbt-pr-review.md index 5d36f09486..1513e7c090 100644 --- a/docs/docs/usage/dbt-pr-review.md +++ b/docs/docs/usage/dbt-pr-review.md @@ -277,6 +277,7 @@ riskTierPathTokens: # OFF by default — see "Risk-tier path tokens" b pii: [ssn, email, phone] # or supply your own case-insensitive tokens allowForceTier: true # set false in gate CI: --force-tier is then rejected checkContractColumns: false # opt-in: new columns missing from an enforced contract (CI that doesn't run `dbt build`) +incrementalSliceVars: [] # opt-in: vars that pick one slice per incremental run (e.g. [tenant]) consumerColumnLists: [] # OFF by default — see "Consumers outside dbt" below rubric: blockOn: [lineage_breakage, contract_violation, pii_exposure, semantic_change] @@ -499,6 +500,31 @@ The deterministic rules group into four families: - **dbt unit tests.** A unit test whose `given` inputs are all `rows: []` is reported: it either never tests the model's logic, or expects rows the model has nothing to build from. +- **Incremental and snapshot semantics.** Config is read from the manifest + (project, yml and inline, as dbt merges them) with the head's inline + `{{ config() }}` on top. Formatting-only edits never count as a change. + - an existing incremental model gains a column, or its `is_incremental()` + filter changes, and the PR doesn't say how existing rows are handled + (`--full-refresh`, a backfill, or forward-only) — a suggestion; + - `partition_by` changes on an incremental model (needs a full refresh); + - `on_schema_change='fail'` and the change adds or removes columns, so the + first incremental run after merge fails; + - a snapshot's `check_cols` or `strategy` changes (`.sql` or dbt 1.9+ yml + snapshots), which can version every row on the next run; + - `row_number()` dedup ordered only by a load timestamp (`loaded_at`, + `ingestion_timestamp`) — a suggestion; loader sync columns such as + `_fivetran_synced` are not flagged. +- **Opt-in: per-slice incremental runs.** For projects that run one slice per + invocation (a tenant or region chosen by a var), list those vars: + `incrementalSliceVars: [tenant]`. A `delete+insert` or `merge` model filtered + by one of them must include the filtered column in `unique_key`, or one + slice's run deletes or overwrites another's rows. +- **Adapter-dependent advice** fires only where the adapter table establishes + the behavior. On Trino, `on_table_exists='replace'` with an enforced contract + runs `create or replace` then `insert`, and `drop` removes the table during + the build, so readers outside dbt can see an empty or missing table (a + suggestion). `unique_key` and strategy changes are covered by the engine's + `DBT011`/`DBT012`. - **dbt config / Jinja (`DBT0xx`).** A core **minijinja** parser reads `{{ config(...) }}` and the model body: incremental model without an `is_incremental()` guard, `merge`/`delete+insert` without a `unique_key`, diff --git a/packages/opencode/src/altimate/review/adapter-capabilities.ts b/packages/opencode/src/altimate/review/adapter-capabilities.ts index cb75636dea..537c6940c7 100644 --- a/packages/opencode/src/altimate/review/adapter-capabilities.ts +++ b/packages/opencode/src/altimate/review/adapter-capabilities.ts @@ -11,6 +11,13 @@ export interface AdapterCapability { /** `int / int` returns an integer, dropping the fraction (7 / 2 = 3). */ integerDivisionTruncates: boolean + /** + * A `table` model with an enforced contract and `on_table_exists='replace'` + * is replaced atomically. False = `create or replace` then `insert` as two + * statements (readers see an empty table), and `on_table_exists='drop'` + * leaves no table during the build. Undefined = not established. + */ + contractTableReplaceAtomic?: boolean /** Documentation backing the entries above. */ sources: string[] } @@ -26,7 +33,14 @@ export const ADAPTER_CAPABILITIES: Record = { }, trino: { integerDivisionTruncates: true, - sources: ["https://trino.io/docs/current/functions/math.html"], + contractTableReplaceAtomic: false, + sources: [ + "https://trino.io/docs/current/functions/math.html", + // create_table_as with an enforced contract: `create or replace table` then `insert`; + // on_table_exists='drop' drops the table before building it. + "https://github.com/starburstdata/dbt-trino/blob/master/dbt/include/trino/macros/adapters.sql", + "https://github.com/starburstdata/dbt-trino/blob/master/dbt/include/trino/macros/materializations/table.sql", + ], }, athena: { // Athena SQL runs on Trino. diff --git a/packages/opencode/src/altimate/review/config.ts b/packages/opencode/src/altimate/review/config.ts index 2dc04c152f..55587f9acd 100644 --- a/packages/opencode/src/altimate/review/config.ts +++ b/packages/opencode/src/altimate/review/config.ts @@ -50,6 +50,12 @@ export const ReviewConfig = z.object({ consumerColumnLists: z .array(z.object({ files: z.string(), pattern: z.string(), model: z.string().optional() })) .default([]), + /** + * Opt-in: vars that select the slice one incremental run processes (e.g. + * `tenant`, `region`). A `delete+insert`/`merge` model filtered by one of + * them must include the filtered column in `unique_key`. + */ + incrementalSliceVars: z.array(z.string()).default([]), /** Allow `--force-tier` (a debug bypass of the risk classifier). Set false * in gate CI so a run cannot skip checks by forcing a lower tier. */ allowForceTier: z.boolean().default(true), diff --git a/packages/opencode/src/altimate/review/dbt-patterns.ts b/packages/opencode/src/altimate/review/dbt-patterns.ts index 1ecc2765dd..91053d7515 100644 --- a/packages/opencode/src/altimate/review/dbt-patterns.ts +++ b/packages/opencode/src/altimate/review/dbt-patterns.ts @@ -249,6 +249,41 @@ const detectDedupTie: Detector = (c) => { }) } +// 9b. Dedup ordered ONLY by a load/ingestion timestamp. Rows loaded in one batch +// share that value, so which version survives is arbitrary (SCD2 versions ranked +// by `ingestion_timestamp` alone). A business column (`updated_at`) stays +// silent, and so do loader sync columns (`_fivetran_synced`, `_airbyte_*`): +// ordering by them is the standard staging idiom and ties are rare per key. +const LOAD_TS_RE = + /^(_?(ingest(ion|ed)?|load(ed)?|etl|extract(ed)?|batch)(_at|_ts|_time|_timestamp|_date|_dt)|_?(ingestion|load|etl|batch)_timestamp|_loaded_at)$/i +const detectDedupLoadTimestamp: Detector = (c) => { + for (const line of c.added) { + const s = stripComments(line) + const m = /row_number\s*\(\s*\)\s*over\s*\(([^)]*)\)/i.exec(s) + if (!m) continue + const order = /order\s+by\s+(.+)$/i.exec(m[1])?.[1] + if (!order) continue + const keys = order.split(",").map((k) => k.trim().replace(/\s+(asc|desc)\b.*$/i, "").replace(/\s+nulls\s+(first|last)$/i, "")) + if (keys.length !== 1) continue + const col = keys[0].split(".").pop()!.replace(/["`]/g, "") + if (!LOAD_TS_RE.test(col)) continue + return makeFinding({ + severity: "suggestion", + category: "dedup", + title: `${c.model}: dedup ordered only by load timestamp \`${col}\``, + body: + `\`row_number()\` is ordered only by \`${col}\`. Rows loaded in the same batch share it, so which version survives is ` + + "arbitrary and can change between runs. Order by the source's own change time first and add a unique tie-breaker.", + file: c.file.path, + model: c.model, + confidence: "high", + evidence: { tool: "dbt-patterns", result: { rule: "dedup_load_timestamp_only", line: line.trim() } }, + ruleKey: "dedup:load-timestamp-only", + }) + } + return null +} + // 11. Function/cast wrapped around a column in an added WHERE — defeats partition pruning. const detectPartitionFunction: Detector = (c) => { const hit = c.added.find((l) => { @@ -735,6 +770,7 @@ const MODEL_DETECTORS: Detector[] = [ detectFanout, detectNotIn, detectDedupTie, + detectDedupLoadTimestamp, detectPartitionFunction, detectCountDistinct, detectPiiIntoMart, diff --git a/packages/opencode/src/altimate/review/incremental-semantics.ts b/packages/opencode/src/altimate/review/incremental-semantics.ts new file mode 100644 index 0000000000..bf7b1991f4 --- /dev/null +++ b/packages/opencode/src/altimate/review/incremental-semantics.ts @@ -0,0 +1,405 @@ +import YAML from "yaml" +import { makeFinding, type Finding } from "./finding" +import type { ChangedFile } from "./diff-filter" +import { capabilityFor } from "./adapter-capabilities" +import { blankComments } from "./select-list" + +/** + * dbt incremental, table-replace and snapshot semantics. These rules follow + * dbt-core behavior, which is the same on every adapter; where the outcome + * depends on the adapter (atomic table replace) they read + * the capability table and stay silent for adapters it doesn't cover. + * + * Config comes from the manifest (dbt_project.yml + yml + inline, merged by + * dbt) overlaid by the head's inline `{{ config() }}` (inline wins in dbt, and + * the manifest may predate the PR). Change detection compares the inline + * config of the two sides; an inline value that moved to yml with the same + * value (per the manifest) is not a change. + * + * All findings are advisory (`dbt-semantics` evidence → catalog origin). + */ + +// --------------------------------------------------------------------------- +// Inline config parsing +// --------------------------------------------------------------------------- + +/** Index of the bracket closing the one at `open`, skipping quoted strings. */ +function closing(s: string, open: number): number { + const pairs: Record = { "(": ")", "[": "]", "{": "}" } + const stack: string[] = [] + let quote: string | null = null + for (let i = open; i < s.length; i++) { + const c = s[i] + if (quote) { + if (c === "\\") i++ + else if (c === quote) quote = null + continue + } + if (c === "'" || c === '"') quote = c + else if (pairs[c]) stack.push(pairs[c]) + else if (c === stack.at(-1)) { + stack.pop() + if (!stack.length) return i + } + } + return -1 +} + +/** Split on commas at nesting depth 0, outside quotes. */ +function topLevelSplit(s: string): string[] { + const out: string[] = [] + let depth = 0 + let quote: string | null = null + let start = 0 + for (let i = 0; i < s.length; i++) { + const c = s[i] + if (quote) { + if (c === "\\") i++ + else if (c === quote) quote = null + continue + } + if (c === "'" || c === '"') quote = c + else if ("([{".includes(c)) depth++ + else if (")]}".includes(c)) depth-- + else if (c === "," && depth === 0) { + out.push(s.slice(start, i)) + start = i + 1 + } + } + out.push(s.slice(start)) + return out +} + +/** `key → raw value text` from every `{{ config(...) }}` in the SQL. */ +export function inlineConfig(sql: string | undefined): Record { + const out: Record = {} + if (!sql) return out + for (const m of sql.matchAll(/\{\{-?\s*config\s*\(/g)) { + const open = m.index! + m[0].length - 1 + const close = closing(sql, open) + if (close < 0) continue + for (const part of topLevelSplit(sql.slice(open + 1, close))) { + const kv = /^\s*([A-Za-z_][A-Za-z0-9_]*)\s*=\s*([\s\S]*?)\s*$/.exec(part) + if (kv) out[kv[1]] = kv[2] + } + } + return out +} + +/** Value text normalized for comparison: no whitespace, one quote style, lowercase. */ +const norm = (v: string | undefined) => v?.replace(/\s+/g, "").replace(/"/g, "'").toLowerCase() +const unquote = (v: string) => v.trim().replace(/^['"]|['"]$/g, "") + +/** An inline config value (Jinja/Python literal text) as data, so it compares + * with the manifest's parsed value; unparseable text stays a string. */ +function inlineValue(raw: string): unknown { + const t = raw.trim() + try { + return JSON.parse( + t.replace(/'/g, '"').replace(/\bTrue\b/g, "true").replace(/\bFalse\b/g, "false").replace(/\bNone\b/g, "null"), + ) + } catch { + return unquote(t) + } +} + +/** Canonical, case-insensitive form of a config value (object keys sorted). */ +function canonValue(v: unknown): string { + if (v === null || typeof v !== "object") return JSON.stringify(typeof v === "string" ? v.toLowerCase() : v) ?? "null" + if (Array.isArray(v)) return "[" + v.map(canonValue).join(",") + "]" + const o = v as Record + return "{" + Object.keys(o).filter((k) => o[k] !== undefined && o[k] !== null).sort().map((k) => `${JSON.stringify(k)}:${canonValue(o[k])}`).join(",") + "}" +} + +/** + * Whether the effective value of `key` changed between the two sides' inline + * config. An inline value removed while the (head) manifest carries the same + * value moved to yml or `dbt_project.yml`: unchanged. A value only the new side + * sets inline can't be compared without the base's effective config, so it is + * not reported. + */ +function inlineConfigChanged( + key: string, + before: Record, + after: Record, + manifest?: Record, +): boolean { + const o = before[key] + const n = after[key] + if (o === undefined || norm(o) === norm(n)) return false + const m = manifest?.[key] + if (n === undefined && m !== undefined && m !== null && canonValue(inlineValue(o)) === canonValue(m)) return false + return true +} + +function asList(v: unknown): string[] | undefined { + if (v === undefined || v === null) return undefined + if (Array.isArray(v)) return v.map((x) => String(x).toLowerCase()) + const s = String(v).trim() + if (s.startsWith("[")) return [...s.matchAll(/['"]([^'"]+)['"]/g)].map((m) => m[1].toLowerCase()) + return [unquote(s).toLowerCase()] +} + +export interface ModelSemantics { + materialized?: string + strategy?: string + uniqueKey?: string[] + onSchemaChange?: string + contractEnforced?: boolean + onTableExists?: string +} + +/** Effective config: manifest `config` overlaid by the inline `{{ config() }}`. */ +export function modelSemantics(inline: Record, manifest?: Record): ModelSemantics { + const m = manifest ?? {} + const str = (k: string) => (inline[k] !== undefined ? unquote(inline[k]).toLowerCase() : typeof m[k] === "string" ? (m[k] as string).toLowerCase() : undefined) + const contractInline = inline.contract !== undefined ? /['"]?enforced['"]?\s*:\s*true/i.test(inline.contract) : undefined + const contractManifest = (m.contract as { enforced?: boolean } | undefined)?.enforced + return { + materialized: str("materialized"), + strategy: str("incremental_strategy"), + uniqueKey: asList(inline.unique_key ?? m.unique_key), + onSchemaChange: str("on_schema_change"), + contractEnforced: contractInline ?? contractManifest, + onTableExists: str("on_table_exists"), + } +} + +/** The SQL with config blocks, comments and whitespace removed: what the model computes. */ +export function modelBody(sql: string): string { + let s = blankComments(sql.replace(/\{#[\s\S]*?#\}/g, " ")) + for (;;) { + const m = /\{\{-?\s*config\s*\(/.exec(s) + if (!m) break + const close = closing(s, m.index + m[0].length - 1) + const end = close < 0 ? -1 : s.indexOf("}}", close) + if (end < 0) break + s = s.slice(0, m.index) + s.slice(end + 2) + } + // Lowercase everything but string literals: `'paid'` → `'PAID'` is a change. + return s + .replace(/\s+/g, " ") + .trim() + .split(/('(?:[^']|'')*')/) + .map((part, k) => (k % 2 ? part : part.toLowerCase())) + .join("") +} + +/** The body as a token stream, so formatting (spaces, line breaks, case) never counts as a change. */ +export function bodyTokens(sql: string): string { + return (modelBody(sql).match(/[a-z0-9_$]+|'(?:[^']|'')*'|[^\s\w]/gi) ?? []).join(" ") +} + +/** The `{% if is_incremental() %}` block(s): what each incremental run picks up. */ +export function incrementalFilter(sql: string): string { + const blocks = [...sql.matchAll(/\{%-?\s*if\s+is_incremental\(\)\s*-?%\}([\s\S]*?)\{%-?\s*endif\s*-?%\}/gi)].map((m) => m[1]) + return blocks.map((b) => bodyTokens(b)).join(" | ") +} + +/** + * Columns compared to one of the named vars: the slice one run processes. Only + * vars the project lists (`incrementalSliceVars`) count, so a filter like + * `currency = '{{ var("reporting_currency") }}'` is not mistaken for a slice. + */ +export function varScopedColumns(sql: string, sliceVars: string[]): string[] { + if (!sliceVars.length) return [] + const names = new Set(sliceVars.map((v) => v.toLowerCase())) + const out = new Set() + const re = /(?:\b[A-Za-z_][\w]*\.)?\b([A-Za-z_][\w]*)\s*(?:=|\bin\b)\s*\(?\s*['"]?\{\{\s*var\s*\(\s*['"]([^'"]+)['"]/gi + for (const m of modelBody(sql).matchAll(re)) if (names.has(m[2].toLowerCase())) out.add(m[1].toLowerCase()) + return [...out] +} + +const PLAN_RE = /full[- _]?refresh|backfill|forward[- ]only|rebuild|re-?run (the )?history|history (is|stays|will)/i + +// --------------------------------------------------------------------------- +// Rules +// --------------------------------------------------------------------------- + +export interface IncrementalInput { + file: ChangedFile + model: string + newSql: string + oldSql?: string + /** The model's `config` from the manifest, when it is there. */ + manifestConfig?: Record + /** Added diff lines (raw). */ + added: string[] + dialect: string + dialectKnown: boolean + /** PR title + description; empty outside a PR run. */ + prText: string + /** Exposures (readers outside dbt) depend on this model. */ + hasExposures: boolean + /** Vars that select the slice one run processes (`incrementalSliceVars`); empty = rule off. */ + sliceVars: string[] + /** Output columns added/removed by the change (lazy: needs column lineage). */ + columnDelta: () => Promise<{ added: string[]; removed: string[] }> +} + +function finding(input: { file: ChangedFile; model: string }, rule: string, severity: Finding["severity"], category: Finding["category"], title: string, body: string): Finding { + return makeFinding({ + severity, + category, + title: `${input.model}: ${title}`, + body, + file: input.file.path, + model: input.model, + confidence: "high", + evidence: { tool: "dbt-semantics", result: { rule } }, + ruleKey: `${category}:${rule}`, + }) +} + +const ticks = (xs: string[]) => xs.map((x) => `\`${x}\``).join(", ") + +export async function incrementalFindings(input: IncrementalInput): Promise { + const inlineNew = inlineConfig(input.newSql) + const inlineOld = inlineConfig(input.oldSql) + const cfg = modelSemantics(inlineNew, input.manifestConfig) + const isNew = input.file.status === "added" + const addedText = input.added.join("\n") + const out: Finding[] = [] + const cap = input.dialectKnown ? capabilityFor(input.dialect) : undefined + + if (cfg.materialized === "incremental") { + // History diverges from new rows when the incremental filter changes or a + // column is added (old rows get NULL). Formatting-only edits never count. + // Under `on_schema_change='fail'` column changes have their own finding + // below, so only a filter change is reported here. + if (!isNew && input.oldSql && bodyTokens(input.oldSql) !== bodyTokens(input.newSql) && + !PLAN_RE.test(input.prText)) { + const filterChanged = incrementalFilter(input.oldSql) !== incrementalFilter(input.newSql) + const added = cfg.onSchemaChange === "fail" ? [] : (await input.columnDelta()).added + if (filterChanged || added.length) { + const why = [ + added.length ? `${ticks(added)} ${added.length === 1 ? "is" : "are"} new: rows built before this change get NULL unless backfilled.` : "", + filterChanged ? "The `is_incremental()` filter changed, so what a run picks up differs from what is already loaded." : "", + ].filter(Boolean).join(" ") + out.push( + finding(input, "incremental-history-plan", "suggestion", "materialization", "incremental change without a plan for existing rows", + `${why} Say in the PR how history is handled: \`--full-refresh\` on deploy, a backfill of the affected range, or that the change is forward-only.`), + ) + } + } + + if (!isNew && inlineConfigChanged("partition_by", inlineOld, inlineNew, input.manifestConfig)) + out.push( + finding(input, "incremental-partition-change", "warning", "materialization", "`partition_by` changed on an incremental model", + "Incremental runs don't rebuild the existing table: depending on the adapter, the next incremental run fails or the table keeps its old partitioning. Run `--full-refresh` on deploy."), + ) + + if (cfg.onSchemaChange === "fail" && !isNew) { + const { added, removed } = await input.columnDelta() + if (added.length || removed.length) { + const what = [added.length ? `adds ${ticks(added)}` : "", removed.length ? `removes ${ticks(removed)}` : ""].filter(Boolean).join(" and ") + out.push( + finding(input, "on-schema-change-fail", "warning", "materialization", "`on_schema_change='fail'` with changed columns", + `This change ${what}, and \`on_schema_change='fail'\` stops the first incremental run after merge. ` + + "Run `--full-refresh` on deploy, or use `append_new_columns` / `sync_all_columns`."), + ) + } + } + + // A per-slice run (filtered by a var) whose key doesn't contain the slice column. + if ((cfg.strategy === "delete+insert" || cfg.strategy === "merge") && cfg.uniqueKey?.length && + (isNew || /unique_key|incremental_strategy|var\s*\(/i.test(addedText))) { + const missing = varScopedColumns(input.newSql, input.sliceVars).filter((c) => !cfg.uniqueKey!.includes(c)) + if (missing.length) + out.push( + finding(input, "unique-key-missing-slice-column", "warning", "dedup", `\`unique_key\` doesn't include ${ticks(missing)}`, + `Each run processes the slice selected by ${ticks(missing)} (a \`var()\` filter), but \`unique_key\` is ${ticks(cfg.uniqueKey)}. ` + + (cfg.strategy === "delete+insert" + ? "A run for one slice deletes other slices' rows that share a key. " + : "A run for one slice overwrites other slices' rows that share a key. ") + + `Add ${ticks(missing)} to \`unique_key\`.`), + ) + } + } + + // Non-atomic table rebuild on an adapter that runs create-then-insert or drop-then-create. + const gap = + cfg.materialized === "table" && cap?.contractTableReplaceAtomic === false + ? cfg.onTableExists === "drop" + ? "drops the table and then builds it, so it doesn't exist during each build" + : cfg.onTableExists === "replace" && cfg.contractEnforced + ? "runs `create or replace table` and then `insert` as two statements (enforced contract), so readers see an empty table during each build" + : undefined + : undefined + if (gap && (isNew || /contract|on_table_exists|materialized/i.test(addedText))) + out.push( + finding(input, "non-atomic-table-replace", "suggestion", "materialization", "table is unavailable while it rebuilds", + `On ${input.dialect}, \`on_table_exists='${cfg.onTableExists}'\` ${gap}. ` + + `Readers outside dbt${input.hasExposures ? " (this model has exposures)" : ""} can hit that window; ` + + "`on_table_exists='rename'` swaps the table in at the end."), + ) + + return out +} + +/** Snapshot definitions: a changed `check_cols`/strategy or SELECT on an existing snapshot. */ +export function snapshotFindings(input: { + file: ChangedFile + model: string + newSql: string + oldSql?: string + prText: string + /** The snapshot's `config` from the manifest, to tell a move to yml from a change. */ + manifestConfig?: Record +}): Finding[] { + if (input.file.status === "added" || !input.oldSql) return [] + const before = inlineConfig(input.oldSql) + const after = inlineConfig(input.newSql) + const out: Finding[] = [] + const changed = ["check_cols", "strategy"].filter((k) => inlineConfigChanged(k, before, after, input.manifestConfig)) + if (changed.length) + out.push( + finding(input, "snapshot-tracking-change", "warning", "materialization", `snapshot ${changed.map((k) => `\`${k}\``).join(" and ")} changed`, + "Changing what an existing snapshot compares can version every row on the next run (a column new to the snapshot compares " + + "NULL with its value), recording history that never happened. Check on a copy first, or start a new snapshot table."), + ) + else if (bodyTokens(input.oldSql) !== bodyTokens(input.newSql) && !PLAN_RE.test(input.prText)) + out.push( + finding(input, "snapshot-logic-no-plan", "warning", "materialization", "snapshot query changed without a plan for history", + "Rows already captured keep the old query's values; the next run versions every row whose values the new query computes differently. " + + "Say in the PR how history is handled (backfill, a new snapshot, or accepted as forward-only)."), + ) + return out +} + +/** dbt 1.9+ snapshots defined in yml (`snapshots:` with `config.check_cols`/`strategy`). */ +export function snapshotYmlFindings(file: ChangedFile, oldContent?: string, newContent?: string): Finding[] { + if (file.status === "added" || file.status === "deleted" || !oldContent || !newContent) return [] + const configs = (text: string) => { + const out = new Map>() + let doc: unknown + try { + doc = YAML.parse(text) + } catch { + return out + } + const list = (doc as { snapshots?: unknown })?.snapshots + if (!Array.isArray(list)) return out + for (const snap of list) + if (snap && typeof snap === "object" && typeof (snap as any).name === "string") + out.set((snap as any).name, ((snap as any).config ?? {}) as Record) + return out + } + const before = configs(oldContent) + const out: Finding[] = [] + for (const [name, cfg] of configs(newContent)) { + const prev = before.get(name) + if (!prev) continue + const changed = ["check_cols", "strategy"].filter( + (k) => prev[k] !== undefined && JSON.stringify(prev[k]) !== JSON.stringify(cfg[k]), + ) + if (changed.length) + out.push( + finding({ file, model: name }, "snapshot-tracking-change", "warning", "materialization", + `snapshot ${changed.map((k) => `\`${k}\``).join(" and ")} changed`, + "Changing what an existing snapshot compares can version every row on the next run (a column new to the snapshot compares " + + "NULL with its value), recording history that never happened. Check on a copy first, or start a new snapshot table."), + ) + } + return out +} diff --git a/packages/opencode/src/altimate/review/orchestrate.ts b/packages/opencode/src/altimate/review/orchestrate.ts index d5b188f9bf..701118b058 100644 --- a/packages/opencode/src/altimate/review/orchestrate.ts +++ b/packages/opencode/src/altimate/review/orchestrate.ts @@ -25,6 +25,8 @@ import { import { detectModelPatterns, detectSchemaYmlPatterns, splitDiff } from "./dbt-patterns" import { listMirrorsModel, type ConsumerColumnList } from "./consumer-lists" import { detectUnitTestFixtures } from "./unit-tests" +import { selectListNames } from "./select-list" +import { incrementalFindings, snapshotFindings, snapshotYmlFindings } from "./incremental-semantics" import { type AiLaneStatus, type AiReviewInput, type AiReviewResult, safeReason } from "./ai-review" /** @@ -153,6 +155,8 @@ export interface ReviewRunner { ): Promise< { exposures: Array<{ name: string; type?: string }>; contractEnforced: boolean; contractColumns: string[] } | undefined > + /** A model's or snapshot's merged dbt `config` from the manifest (project + yml + inline). */ + modelConfig?(name: string): Promise | undefined> /** Column-lineage edges for a query: source `"table"."col"` → target output column. */ columnLineage?(sql: string, dialect: string): Promise> /** Classify a set of column names as PII (Email/SSN/…) with confidence + masking. */ @@ -631,13 +635,23 @@ async function columnBreakageLane( ] } -/** A query's output columns from column lineage (lowercased); empty when unknown or `*`. */ +/** + * A query's output columns (lowercased); empty when unknown or `*`. Column + * lineage gives columns that read a source column; the final SELECT's own + * aliases add literal and function-only columns lineage can't see. + */ async function outputColumns(runner: ReviewRunner, sql: string, dialect: string): Promise> { if (!runner.columnLineage) return new Set() const cols = (await runner.columnLineage(sql, dialect)).map((e) => e.target.toLowerCase()).filter(Boolean) // A star in the output means the list is incomplete: report nothing rather than guess. if (cols.some((c) => c === "*" || c.endsWith(".*"))) return new Set() - return new Set(cols) + // Lineage returned nothing (failed, or could not analyse this side): the + // set is unknown. Filling it from the select list alone would make every + // unaliased expression look removed when only one side's lineage failed. + if (!cols.length) return new Set() + // `*` in the final SELECT makes `named` null: keep lineage's own answer. + const named = selectListNames(sql) + return new Set([...cols, ...(named ?? [])]) } /** Output columns added and removed by the change (both empty when either side is unknown). */ @@ -817,6 +831,35 @@ async function consumerListLane( return out } +/** Incremental / table-replace semantics (incremental-semantics.ts). */ +async function incrementalSemanticsLane( + ctx: ModelContext, + runner: ReviewRunner, + dialect: string, + dialectKnown: boolean, + prText: string, + sliceVars: string[], +): Promise { + const { file, newSql, oldSql } = ctx + if (!newSql || file.status === "deleted") return [] + const model = modelNameFromPath(file.path) + const consumers = await runner.modelConsumers?.(model) + return incrementalFindings({ + file, + model, + newSql, + oldSql: hasOldSide(file) ? oldSql : undefined, + manifestConfig: await runner.modelConfig?.(model), + added: splitDiff(file.diff).added, + dialect, + dialectKnown, + prText, + hasExposures: !!consumers?.exposures.length, + sliceVars, + columnDelta: () => columnDelta(ctx, runner, dialect), + }) +} + /** * DATA-DIFF — the showcase: actually run base vs head against the warehouse * (core DataParity) and report row/value deltas a linter can never see. Needs a @@ -1488,6 +1531,7 @@ export async function runReview(input: OrchestrateInput): Promise | undefined const changedYmlText = () => @@ -1596,6 +1640,9 @@ export async function runReview(input: OrchestrateInput): Promise f.kind === "snapshot" && f.status !== "deleted" && f.status !== "added") + const sets = await Promise.all( + snapshots.map(async (file) => { + try { + const [oldSql, newSql] = await Promise.all([getContent?.(file.oldPath ?? file.path, "old"), getContent?.(file.path, "new")]) + if (!newSql) return [] + const model = modelNameFromPath(file.path) + const manifestConfig = await input.runner.modelConfig?.(model) + return snapshotFindings({ file, model, newSql, oldSql, prText, manifestConfig }) + } catch (err) { + failedFile(file, err) + return [] + } + }), + ) + for (const findings of sets) all.push(findings) + } + // schema.yml-level detectors (test removal) — run on changed YAML files // regardless of tier, since deleting a guardrail test is always worth flagging. // Pass old + new content when available so the detector diffs structural YAML @@ -1670,6 +1737,7 @@ export async function runReview(input: OrchestrateInput): Promise = { + BigQuery: "bigquery", + Snowflake: "snowflake", + Redshift: "redshift", + Databricks: "databricks", + // Spark SQL runs on the Databricks/Spark adapters (the `spark` adapter maps to `databricks`). + Spark: "databricks", + "Spark/Databricks": "databricks", +} + +/** The engine a dialect-pack rule is about (from its `Engine:` title prefix), if any. */ +export function ruleEngine(rule: Rule): string | undefined { + const m = /^(BigQuery|Snowflake|Redshift|Databricks|Spark\/Databricks|Spark):/.exec(rule.title) + return m ? PACK_ENGINE[m[1]] : undefined +} + /** Evaluate the catalog against a changed file. */ export function evaluateCatalog( file: ChangedFile, @@ -152,6 +168,10 @@ export function evaluateCatalog( const model = modelName(file.path) const matched: { rule: Rule; finding: Finding }[] = [] for (const rule of CATALOG) { + // A dialect pack (BigQuery partition filters, Snowflake VARIANT…) only + // applies to its own engine once the project's engine is known. + const engine = ruleEngine(rule) + if (engine && ctx?.dialectKnown && ctx.dialect.toLowerCase().replace(/^spark$/, "databricks") !== engine) continue if (!ruleMatches(rule, input)) continue const body = gatedBody(rule, ctx) if (body === null) continue diff --git a/packages/opencode/src/altimate/review/runner.ts b/packages/opencode/src/altimate/review/runner.ts index 71107086fe..29a9e4e83e 100644 --- a/packages/opencode/src/altimate/review/runner.ts +++ b/packages/opencode/src/altimate/review/runner.ts @@ -449,6 +449,19 @@ export function createDispatcherRunner(opts: DispatcherRunnerOptions): ReviewRun } }, + async modelConfig(name: string) { + try { + const raw = loadRawManifest(opts.manifestPath) + const hit = Object.entries(raw?.nodes ?? {}).find( + ([id, n]) => (id.startsWith("model.") || id.startsWith("snapshot.")) && n?.name === name, + ) + return hit?.[1]?.config as Record | undefined + } catch (err) { + fail("modelConfig", err) + return undefined + } + }, + async isComplex(sql: string): Promise { try { const res = await callCore("altimate_core.metadata", { sql }) diff --git a/packages/opencode/src/altimate/review/select-list.ts b/packages/opencode/src/altimate/review/select-list.ts new file mode 100644 index 0000000000..9536bcc1bd --- /dev/null +++ b/packages/opencode/src/altimate/review/select-list.ts @@ -0,0 +1,98 @@ +/** + * Output column names of a query's final SELECT, read from the SQL text. + * Column lineage only reports columns that read a source column, so literal + * and function-only columns (`current_date as load_date`, `1 as one`) are + * invisible to it; this fills that gap. Conservative by design: an item counts + * only when it is aliased (`… as name` / `… name`) or a bare (qualified) + * column; anything else contributes nothing. `*` makes the list unknown. + */ + +/** Text with `--` and block comments blanked, same length. Quoted strings and + * quoted identifiers are skipped first, so `'x--y'` is not a comment. */ +export function blankComments(sql: string): string { + let out = "" + let i = 0 + while (i < sql.length) { + const c = sql[i] + if (c === "'" || c === '"' || c === "`") { + // A doubled quote inside a string is an escaped quote. + let j = i + 1 + while (j < sql.length && !(sql[j] === c && sql[j + 1] !== c)) j += sql[j] === c ? 2 : 1 + out += sql.slice(i, j + 1) + i = j + 1 + } else if (c === "-" && sql[i + 1] === "-") { + const end = sql.indexOf("\n", i) + const stop = end < 0 ? sql.length : end + out += " ".repeat(stop - i) + i = stop + } else if (c === "/" && sql[i + 1] === "*") { + const end = sql.indexOf("*/", i + 2) + const stop = end < 0 ? sql.length : end + 2 + out += sql.slice(i, stop).replace(/[^\n]/g, " ") + i = stop + } else { + out += c + i++ + } + } + return out +} + +/** Text with comments, quoted strings and parenthesized groups blanked, same length. */ +function flatten(sql: string): string { + let s = blankComments(sql) + s = s.replace(/'(?:[^']|'')*'/g, (m) => "'" + "_".repeat(m.length - 2) + "'") + let out = "" + let depth = 0 + for (const c of s) { + if (c === "(") { + depth++ + out += "(" + } else if (c === ")") { + depth = Math.max(0, depth - 1) + out += ")" + } else out += depth > 0 ? " " : c + } + return out +} + +const IDENT = /^[A-Za-z_][A-Za-z0-9_$]*$/ +const unquote = (s: string) => s.replace(/^["`\[]|["`\]]$/g, "") + +/** A top-level set operator (parenthesized groups are already blanked). */ +const SET_OP = /\b(union|intersect|except|minus)\b(\s+(all|distinct))?\s*(\bselect\b|\()/i + +/** Output names of the final top-level SELECT, or null when unknown (`*`, unparseable). + * A set operation (`UNION` …) takes its names from its first branch. */ +export function selectListNames(sql: string): string[] | null { + const flat = flatten(sql) + const selects = [...flat.matchAll(/\bselect\b/gi)] + if (!selects.length) return null + const head = SET_OP.test(flat) ? selects[0] : selects[selects.length - 1] + const start = head.index! + "select".length + const fromRel = flat.slice(start).search(/\bfrom\b/i) + const listFlat = flat.slice(start, fromRel < 0 ? undefined : start + fromRel) + const listRaw = blankComments(sql).slice(start, fromRel < 0 ? undefined : start + fromRel) + // Split on top-level commas (the flattened text has no nested commas). + const names: string[] = [] + let pos = 0 + for (const piece of listFlat.split(",")) { + const raw = listRaw.slice(pos, pos + piece.length) + pos += piece.length + 1 + const item = piece.replace(/^\s*(distinct|all)\s+/i, "").trim() + if (!item) continue + if (/(^|\.)\*$/.test(item)) return null + const rawItem = raw.trim().replace(/^(distinct|all)\s+/i, "") + const bare = /^(?:[A-Za-z_][A-Za-z0-9_$]*\.)*("[^"]+"|[A-Za-z_][A-Za-z0-9_$]*)$/.exec(rawItem) + let name = bare ? unquote(bare[1]) : undefined + if (!name) { + const m = /\s(as\s+)?("[^"]+"|`[^`]+`|[A-Za-z_][A-Za-z0-9_$]*)\s*$/i.exec(rawItem) + const before = m ? rawItem.slice(0, m.index).trimEnd() : "" + // An alias follows a complete expression: `… as x`, or `expr x` where expr + // ends in an identifier, `)`, a literal or `end` (never `a + b`). + if (m && before && (m[1] || /([A-Za-z0-9_$"`\]')]|\bend)$/i.test(before))) name = unquote(m[2]) + } + if (name && (IDENT.test(name) || /^[^"`]+$/.test(name)) && !/^(end|null|true|false)$/i.test(name)) names.push(name.toLowerCase()) + } + return names +} diff --git a/packages/opencode/test/altimate/review-contracts-dialect.test.ts b/packages/opencode/test/altimate/review-contracts-dialect.test.ts index 6632c130f6..f09630551a 100644 --- a/packages/opencode/test/altimate/review-contracts-dialect.test.ts +++ b/packages/opencode/test/altimate/review-contracts-dialect.test.ts @@ -323,3 +323,35 @@ describe("unit tests that cannot exercise the model", () => { expect(detectUnitTestFixtures(file, yml("[]"), yml("[]"))).toHaveLength(0) }) }) + +describe("dialect packs apply to their own engine", () => { + const file = { path: "models/marts/fct_orders.sql", status: "added" as const } + const sql = "select count(distinct user_id) from `proj`.`ds`.`events_fct`" + const packs = (ctx?: { dialect: string; dialectKnown: boolean }) => + evaluateCatalog(file, sql, [sql], [], DEFAULT_RUBRIC, ctx) + .map((f) => f.title) + .filter((t) => /: (BigQuery|Snowflake|Redshift|Databricks):/.test(t)) + + test("a DuckDB or Snowflake project gets no BigQuery advice", () => { + expect(packs({ dialect: "duckdb", dialectKnown: true })).toEqual([]) + expect(packs({ dialect: "snowflake", dialectKnown: true })).toEqual([]) + }) + + test("a BigQuery project, or an unknown engine, keeps it", () => { + expect(packs({ dialect: "bigquery", dialectKnown: true }).length).toBeGreaterThan(0) + expect(packs({ dialect: "snowflake", dialectKnown: false }).length).toBeGreaterThan(0) + expect(packs().length).toBeGreaterThan(0) + }) + test("Spark rules fire only on Databricks/Spark once the engine is known", () => { + const line = "select to_date(order_date), explode(items) from t" + const spark = (ctx?: { dialect: string; dialectKnown: boolean }) => + evaluateCatalog(file, line, [line], [], DEFAULT_RUBRIC, ctx) + .map((f) => f.title) + .filter((t) => /: Spark(\/Databricks)?:/.test(t)) + expect(spark({ dialect: "snowflake", dialectKnown: true })).toEqual([]) + expect(spark({ dialect: "postgres", dialectKnown: true })).toEqual([]) + expect(spark({ dialect: "databricks", dialectKnown: true })).toHaveLength(2) + expect(spark({ dialect: "spark", dialectKnown: true })).toHaveLength(2) + expect(spark()).toHaveLength(2) + }) +}) diff --git a/packages/opencode/test/altimate/review-incremental-semantics.test.ts b/packages/opencode/test/altimate/review-incremental-semantics.test.ts new file mode 100644 index 0000000000..bbf424ceb6 --- /dev/null +++ b/packages/opencode/test/altimate/review-incremental-semantics.test.ts @@ -0,0 +1,301 @@ +import { describe, test, expect } from "bun:test" +import { + incrementalFindings, + inlineConfig, + modelBody, + modelSemantics, + snapshotFindings, + snapshotYmlFindings, + varScopedColumns, + type IncrementalInput, +} from "../../src/altimate/review/incremental-semantics" +import { detectModelPatterns } from "../../src/altimate/review/dbt-patterns" +import { runReview, type ReviewRunner } from "../../src/altimate/review/orchestrate" +import { DEFAULT_REVIEW_CONFIG } from "../../src/altimate/review/config" +import { DEFAULT_RUBRIC } from "../../src/altimate/review/rubric" + +/** + * dbt incremental / microbatch / snapshot semantics. Generic rules are checked + * on two adapters; adapter-dependent rules fire only where the capability + * table says the behavior is real. + */ + +const INC = (body: string, cfg = "materialized='incremental', incremental_strategy='merge', unique_key='id'") => + `{{ config(${cfg}) }}\n${body}\n{% if is_incremental() %} where ts > (select max(ts) from {{ this }}) {% endif %}\n` + +function input(over: Partial): IncrementalInput { + return { + file: { path: "models/marts/fct.sql", status: "modified" }, + model: "fct", + newSql: INC("select id, ts from src"), + oldSql: INC("select id, ts from src"), + added: [], + dialect: "snowflake", + dialectKnown: true, + prText: "", + hasExposures: false, + sliceVars: [], + columnDelta: async () => ({ added: [], removed: [] }), + ...over, + } +} +const rules = async (over: Partial) => + (await incrementalFindings(input(over))).map((f) => (f.evidence?.result as { rule: string }).rule) + +describe("config parsing", () => { + test("inline config keeps nested values whole", () => { + const c = inlineConfig("{{ config(materialized='incremental', unique_key=['a', 'b'], contract={'enforced': true}, partition_by={'field': 'd', 'granularity': 'day'}) }}") + expect(c.unique_key).toBe("['a', 'b']") + expect(c.partition_by).toBe("{'field': 'd', 'granularity': 'day'}") + expect(modelSemantics(c).uniqueKey).toEqual(["a", "b"]) + expect(modelSemantics(c).contractEnforced).toBe(true) + }) + + test("manifest config fills what the inline config doesn't set; inline wins", () => { + const s = modelSemantics(inlineConfig("{{ config(on_schema_change='fail') }}"), { + materialized: "incremental", + on_schema_change: "ignore", + unique_key: "id", + }) + expect(s).toMatchObject({ materialized: "incremental", onSchemaChange: "fail", uniqueKey: ["id"] }) + }) + + test("the model body keeps string literals: `--` inside one and its case both count", () => { + expect(modelBody("select 'a--b' as x, y from t")).toContain("y from t") + expect(modelBody("select * from t where s = 'paid'")).not.toBe(modelBody("select * from t where s = 'PAID'")) + expect(modelBody("SELECT * FROM t")).toBe(modelBody("select * from t")) + }) + + test("the model body ignores config, comments and whitespace", () => { + expect(modelBody("{{ config(a='b') }}\nselect 1 -- x\n")).toBe(modelBody("{# doc #}{{ config(a='c') }} select 1")) + }) + + test("only columns compared to a listed slice var count", () => { + const sql = "select * from t where t.tenant_id = '{{ var(\"tenant\") }}' and currency = '{{ var('reporting_currency') }}'" + expect(varScopedColumns(sql, ["tenant"])).toEqual(["tenant_id"]) + expect(varScopedColumns(sql, [])).toEqual([]) + }) +}) + +describe("incremental change without a plan for existing rows", () => { + const delta = (added: string[]) => async () => ({ added, removed: [] }) + for (const dialect of ["snowflake", "trino"]) { + test(`a new column fires on ${dialect}; a stated plan silences it`, async () => { + const changed = { dialect, newSql: INC("select id, ts, kind from src"), columnDelta: delta(["kind"]) } + const [f] = await incrementalFindings(input(changed)) + expect(f.body).toContain("`kind` is new: rows built before this change get NULL") + expect(f.severity).toBe("suggestion") + expect(await rules({ ...changed, prText: "Needs a --full-refresh on deploy" })).not.toContain("incremental-history-plan") + }) + } + + test("a changed is_incremental() filter fires", async () => { + const oldSql = "{{ config(materialized='incremental', unique_key='id') }} select id, ts from src {% if is_incremental() %} where ts > (select max(ts) from {{ this }}) {% endif %}" + const newSql = oldSql.replace("where ts >", "where ts >=") + expect(await rules({ oldSql, newSql })).toContain("incremental-history-plan") + }) + + test("formatting-only edits, expression edits without new columns, config-only and new models don't fire", async () => { + expect(await rules({ newSql: INC("SELECT id ,\n ts\nFROM src") })).toHaveLength(0) + expect(await rules({ newSql: INC("select id, ts + 1 as ts from src") })).toHaveLength(0) + expect(await rules({ newSql: INC("select id, ts from src", "materialized='incremental', incremental_strategy='merge', unique_key='id', tags=['x']") })).toHaveLength(0) + expect(await rules({ file: { path: "models/marts/fct.sql", status: "added" }, oldSql: undefined, newSql: INC("select 2"), columnDelta: delta(["x"]) })).toHaveLength(0) + }) + + test("partition_by change needs a full refresh", async () => { + const cfg = (g: string) => `materialized='incremental', unique_key='id', partition_by={'field': 'd', 'granularity': '${g}'}` + expect(await rules({ oldSql: INC("select 1", cfg("day")), newSql: INC("select 1", cfg("month")) })).toContain("incremental-partition-change") + }) + + test("partition_by moved from inline config to yml with the same value is not a change", async () => { + const withPart = "materialized='incremental', unique_key='id', partition_by={'field': 'd', 'granularity': 'day'}" + const without = "materialized='incremental', unique_key='id'" + const moved = { field: "d", granularity: "day" } + expect( + await rules({ oldSql: INC("select 1", withPart), newSql: INC("select 1", without), manifestConfig: { materialized: "incremental", partition_by: moved } }), + ).not.toContain("incremental-partition-change") + // Removed inline and not in yml either: the partitioning really changed. + expect( + await rules({ oldSql: INC("select 1", withPart), newSql: INC("select 1", without), manifestConfig: { materialized: "incremental" } }), + ).toContain("incremental-partition-change") + }) +}) + +describe("on_schema_change='fail'", () => { + const cfg = "materialized='incremental', unique_key='id', on_schema_change='fail'" + test("a changed column set fails the first incremental run", async () => { + const out = await incrementalFindings( + input({ oldSql: INC("select id from s", cfg), newSql: INC("select id, kind from s", cfg), prText: "full refresh", columnDelta: async () => ({ added: ["kind"], removed: [] }) }), + ) + expect(out.map((f) => f.body).join()).toContain("adds `kind`") + }) + test("it replaces, not repeats, the history-plan finding", async () => { + const out = await rules({ oldSql: INC("select id from s", cfg), newSql: INC("select id, kind from s", cfg), columnDelta: async () => ({ added: ["kind"], removed: [] }) }) + expect(out).toEqual(["on-schema-change-fail"]) + }) + test("an unchanged column set is fine", async () => { + expect(await rules({ oldSql: INC("select id from s", cfg), newSql: INC("select id from s", cfg) })).not.toContain("on-schema-change-fail") + }) + test("a filter-only change is still reported as needing a history plan", async () => { + const oldSql = `{{ config(${cfg}) }} select id from s {% if is_incremental() %} where ts > (select max(ts) from {{ this }}) {% endif %}` + const newSql = oldSql.replace("where ts >", "where ts >=") + expect(await rules({ oldSql, newSql })).toEqual(["incremental-history-plan"]) + }) +}) + +describe("per-slice runs whose unique_key lacks the slice column", () => { + const sql = (key: string, strategy = "delete+insert") => + `{{ config(materialized='incremental', incremental_strategy='${strategy}', unique_key=${key}) }}\nselect event_id, tenant_id from src where tenant_id = '{{ var("tenant") }}'` + for (const dialect of ["snowflake", "postgres"]) { + test(`fires on ${dialect} for delete+insert and merge`, async () => { + const isNew = { file: { path: "models/marts/fct.sql", status: "added" as const }, oldSql: undefined, dialect, sliceVars: ["tenant"] } + const [di] = await incrementalFindings(input({ ...isNew, newSql: sql("'event_id'") })) + expect(di.body).toContain("deletes other slices' rows") + const [mg] = await incrementalFindings(input({ ...isNew, newSql: sql("'event_id'", "merge") })) + expect(mg.body).toContain("overwrites other slices' rows") + expect(await rules({ ...isNew, newSql: sql("['event_id', 'tenant_id']") })).toHaveLength(0) + expect(await rules({ ...isNew, sliceVars: [], newSql: sql("'event_id'") })).toHaveLength(0) + }) + } +}) + +describe("non-atomic table rebuild", () => { + const tbl = (ote: string, contract = true) => + `{{ config(materialized='table', on_table_exists='${ote}'${contract ? ", contract={'enforced': true}" : ""}) }}\nselect 1 as id` + const isNew = { file: { path: "models/marts/fct.sql", status: "added" as const }, oldSql: undefined, dialect: "trino" } + test("trino: replace with an enforced contract, or drop; always a suggestion", async () => { + const [f] = await incrementalFindings(input({ ...isNew, newSql: tbl("replace") })) + expect(f.severity).toBe("suggestion") + expect(f.body).toContain("`create or replace table` and then `insert`") + const [e] = await incrementalFindings(input({ ...isNew, newSql: tbl("replace"), hasExposures: true })) + expect(e.severity).toBe("suggestion") + expect(e.body).toContain("this model has exposures") + expect((await incrementalFindings(input({ ...isNew, newSql: tbl("drop", false) })))[0].body).toContain("doesn't exist during each build") + }) + test("rename, replace without a contract, or another adapter: silent", async () => { + expect(await rules({ ...isNew, newSql: tbl("rename") })).toHaveLength(0) + expect(await rules({ ...isNew, newSql: tbl("replace", false) })).toHaveLength(0) + expect(await rules({ ...isNew, dialect: "snowflake", newSql: tbl("replace") })).toHaveLength(0) + }) +}) + +describe("snapshots", () => { + const snap = (cols: string, body = "select id, status from src") => + `{% snapshot s %}\n{{ config(unique_key='id', strategy='check', check_cols=${cols}) }}\n${body}\n{% endsnapshot %}` + const file = { path: "snapshots/s.sql", status: "modified" as const } + const rule = (f: ReturnType) => f.map((x) => (x.evidence?.result as { rule: string }).rule) + test("a check_cols change can record history that never happened", () => { + expect(rule(snapshotFindings({ file, model: "s", oldSql: snap("['status']"), newSql: snap("['status', 'amount']"), prText: "" }))).toEqual([ + "snapshot-tracking-change", + ]) + }) + test("check_cols moved from inline config to yml with the same value is not a tracking change", () => { + const inline = `{% snapshot s %}\n{{ config(unique_key='id', strategy='check', check_cols=['status']) }}\nselect id, status from src\n{% endsnapshot %}` + const bare = `{% snapshot s %}\n{{ config(unique_key='id') }}\nselect id, status from src\n{% endsnapshot %}` + const moved = snapshotFindings({ file, model: "s", oldSql: inline, newSql: bare, prText: "", manifestConfig: { strategy: "check", check_cols: ["status"] } }) + expect(rule(moved)).toEqual([]) + const changed = snapshotFindings({ file, model: "s", oldSql: inline, newSql: bare, prText: "", manifestConfig: { strategy: "check", check_cols: ["status", "amount"] } }) + expect(rule(changed)).toEqual(["snapshot-tracking-change"]) + }) + + test("yml-defined snapshots (dbt 1.9+) are checked too", () => { + const y = (cols: string) => `snapshots:\n - name: orders_snap\n relation: source('raw', 'orders')\n config:\n unique_key: id\n strategy: check\n check_cols: ${cols}\n` + const yfile = { path: "snapshots/_snapshots.yml", status: "modified" as const } + expect(snapshotYmlFindings(yfile, y("[status]"), y("[status, amount]")).map((f) => f.title)).toEqual(["orders_snap: snapshot `check_cols` changed"]) + expect(snapshotYmlFindings(yfile, y("[status]"), y("[status]"))).toEqual([]) + }) + + test("a query change needs a plan; a new snapshot is fine", () => { + const changed = { file, model: "s", oldSql: snap("['status']"), newSql: snap("['status']", "select id, upper(status) as status from src") } + expect(rule(snapshotFindings({ ...changed, prText: "" }))).toEqual(["snapshot-logic-no-plan"]) + expect(rule(snapshotFindings({ ...changed, prText: "we will backfill" }))).toEqual([]) + expect(rule(snapshotFindings({ ...changed, newSql: snap("['status']", "SELECT id,\n status FROM src"), prText: "" }))).toEqual([]) + expect(snapshotFindings({ ...changed, file: { ...file, status: "added" }, prText: "" })).toEqual([]) + }) +}) + +describe("dedup ordered only by a load timestamp", () => { + const run = (line: string) => + detectModelPatterns({ path: "models/m.sql", status: "added", diff: `+${line}\n` }, line, DEFAULT_RUBRIC).filter( + (f) => (f.evidence?.result as { rule?: string })?.rule === "dedup_load_timestamp_only", + ) + test("load timestamps fire, as a suggestion", () => { + for (const col of ["loaded_at", "ingestion_timestamp", "src.load_ts", "_loaded_at"]) { + const f = run(`select *, row_number() over (partition by id order by ${col} desc) as rn from t`) + expect(f).toHaveLength(1) + expect(f[0].severity).toBe("suggestion") + } + }) + test("loader sync columns (the standard staging idiom) stay silent", () => { + for (const col of ["_fivetran_synced", "_airbyte_extracted_at", "_sdc_batched_at"]) + expect(run(`select *, row_number() over (partition by id order by ${col} desc) as rn from t`)).toHaveLength(0) + }) + test("a business timestamp, a tie-breaker, or no row_number: silent", () => { + expect(run("select *, row_number() over (partition by id order by updated_at desc) as rn from t")).toHaveLength(0) + expect(run("select *, row_number() over (partition by id order by loaded_at desc, seq desc) as rn from t")).toHaveLength(0) + expect(run("select *, rank() over (partition by id order by loaded_at) as rn from t")).toHaveLength(0) + }) +}) + +describe("end to end through the orchestrator", () => { + const runner = { + async impact() { + return { hasManifest: true, severity: "SAFE", directCount: 0, transitiveCount: 0, testCount: 0 } + }, + async grade() { + return { grade: "B" } + }, + async check() { + return { issues: [], ran: true } + }, + async equivalence() { + return { decided: true, equivalent: false } + }, + async detectPii() { + return { columns: [] } + }, + async modelConfig() { + return { materialized: "incremental", unique_key: "id" } + }, + async columnLineage(sql: string) { + const m = /select\s+([\s\S]*?)\s+from/i.exec(sql) + return m ? m[1].split(",").map((c) => ({ source: `"src"."${c.trim()}"`, target: c.trim() })) : [] + }, + } as ReviewRunner + + test("lineage that returns nothing for one side leaves the column set unknown, not every expression removed", async () => { + const env = await runReview({ + changedFiles: [{ path: "models/marts/fct.sql", status: "modified", diff: "-select id, upper(a) as a_up from src\n+select id, upper(a) as a_up, kind from src\n" }], + config: { ...DEFAULT_REVIEW_CONFIG, reviewers: ["dbt_patterns"] }, + rubric: DEFAULT_RUBRIC, + mode: "gate", + runner: { ...runner, async columnLineage(sql: string) { return sql.includes("kind") ? runner.columnLineage!(sql, "duckdb") : [] } } as ReviewRunner, + getContent: async (_f, side) => (side === "old" ? "select id, upper(a) as a_up from src" : "select id, upper(a) as a_up, kind from src"), + }) + expect(env.findings.some((f) => f.title.includes("incremental change without a plan"))).toBe(false) + }) + + test("an incremental model whose materialization is set in yml (manifest) still gets the rule; findings never block", async () => { + const env = await runReview({ + changedFiles: [ + { path: "models/marts/fct.sql", status: "modified", diff: "-select id from src\n+select id, kind from src\n" }, + { path: "snapshots/s.sql", status: "modified", diff: "-check_cols=['a']\n+check_cols=['a', 'b']\n" }, + ], + config: { ...DEFAULT_REVIEW_CONFIG, reviewers: ["dbt_patterns"] }, + rubric: DEFAULT_RUBRIC, + mode: "gate", + runner, + getContent: async (f, side) => + f.startsWith("snapshots/") + ? `{{ config(strategy='check', unique_key='id', check_cols=${side === "old" ? "['a']" : "['a', 'b']"}) }} select id, a, b from src` + : side === "old" + ? "select id from src" + : "select id, kind from src", + }) + const titles = env.findings.map((f) => f.title) + expect(titles).toContain("fct: incremental change without a plan for existing rows") + expect(titles.some((t) => t.startsWith("s: snapshot `check_cols` changed"))).toBe(true) + expect(env.findings.filter((f) => f.evidence?.tool === "dbt-semantics").every((f) => f.origin === "catalog")).toBe(true) + expect(env.verdict).not.toBe("REQUEST_CHANGES") + }) +}) diff --git a/packages/opencode/test/altimate/review-select-list.test.ts b/packages/opencode/test/altimate/review-select-list.test.ts new file mode 100644 index 0000000000..a61aaf2d25 --- /dev/null +++ b/packages/opencode/test/altimate/review-select-list.test.ts @@ -0,0 +1,29 @@ +import { describe, test, expect } from "bun:test" +import { selectListNames } from "../../src/altimate/review/select-list" + +describe("final SELECT output names", () => { + test("aliases, bare and qualified columns, literals and functions", () => { + expect( + selectListNames("select o.id, lower(kind) as kind, current_date as load_date, 1 as one, count(*) cnt, case when a then 1 end flag from t o"), + ).toEqual(["id", "kind", "load_date", "one", "cnt", "flag"]) + }) + test("the last top-level SELECT wins over CTEs and subqueries", () => { + expect(selectListNames("with a as (select x, y from t) select a.x, (select max(z) from u) as mz from a")).toEqual(["x", "mz"]) + }) + test("a set operation takes its names from the first branch", () => { + expect(selectListNames("select id, a as kind from t union all select id, b as other from u")).toEqual(["id", "kind"]) + expect(selectListNames("with c as (select x from t) select x as y from c except select z from d")).toEqual(["y"]) + }) + test("ambiguous items contribute nothing; operators are not aliases", () => { + expect(selectListNames("select a + b, c - 1 as d, case when e then 1 end from t")).toEqual(["d"]) + }) + test("a star makes the list unknown; comments and strings don't confuse it", () => { + expect(selectListNames("select t.*, x from t")).toBeNull() + expect(selectListNames("select 'a, b' as s, -- , trailing\n y from t")).toEqual(["s", "y"]) + expect(selectListNames("select distinct id, name from t")).toEqual(["id", "name"]) + }) + test("`--` inside a string literal is not a comment", () => { + expect(selectListNames("select id, 'x--y' as tag, z from t")).toEqual(["id", "tag", "z"]) + expect(selectListNames("select id, 'it''s -- fine' as note /* c, d */ , z from t")).toEqual(["id", "note", "z"]) + }) +})