Repository navigation
feat(review): consumers outside dbt, engine-aware integer division, opt-in contract and column-list checks #1724
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: fix/review-failures-gate
Are you sure you want to change the base?
Changes from all commits
dc5390b
cb2feda
41e2bd8
9fa035e
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,79 @@ | ||
| /** | ||
| * Per-adapter behavior that review rules depend on. A rule whose correctness | ||
| * differs by engine reads this table instead of hard-coding one engine, so it | ||
| * fires where the behavior is real and stays silent elsewhere. Every entry | ||
| * cites the engine's documentation. An adapter missing from the table gets the | ||
| * generic (engine-agnostic) form of each rule only. | ||
| * | ||
| * Keys are the review's dialect names (see run.ts `ADAPTER_DIALECT`). | ||
| */ | ||
|
|
||
| export interface AdapterCapability { | ||
| /** `int / int` returns an integer, dropping the fraction (7 / 2 = 3). */ | ||
| integerDivisionTruncates: boolean | ||
| /** Documentation backing the entries above. */ | ||
| sources: string[] | ||
| } | ||
|
|
||
| export const ADAPTER_CAPABILITIES: Record<string, AdapterCapability> = { | ||
| postgres: { | ||
| integerDivisionTruncates: true, | ||
| sources: ["https://www.postgresql.org/docs/current/functions-math.html"], | ||
| }, | ||
| redshift: { | ||
| integerDivisionTruncates: true, | ||
| sources: ["https://docs.aws.amazon.com/redshift/latest/dg/r_numeric_computations201.html"], | ||
| }, | ||
| trino: { | ||
| integerDivisionTruncates: true, | ||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. WARNING: Trino/Athena Trino's |
||
| sources: ["https://trino.io/docs/current/functions/math.html"], | ||
| }, | ||
| athena: { | ||
| // Athena SQL runs on Trino. | ||
| integerDivisionTruncates: true, | ||
| sources: ["https://trino.io/docs/current/functions/math.html"], | ||
| }, | ||
| tsql: { | ||
| integerDivisionTruncates: true, | ||
| sources: ["https://learn.microsoft.com/en-us/sql/t-sql/language-elements/divide-transact-sql"], | ||
| }, | ||
| fabric: { | ||
| integerDivisionTruncates: true, | ||
| sources: ["https://learn.microsoft.com/en-us/sql/t-sql/language-elements/divide-transact-sql"], | ||
| }, | ||
| snowflake: { | ||
| integerDivisionTruncates: false, | ||
| sources: ["https://docs.snowflake.com/en/sql-reference/operators-arithmetic"], | ||
| }, | ||
| bigquery: { | ||
| integerDivisionTruncates: false, | ||
| sources: ["https://cloud.google.com/bigquery/docs/reference/standard-sql/operators#arithmetic_operators"], | ||
| }, | ||
| databricks: { | ||
| integerDivisionTruncates: false, | ||
| sources: ["https://docs.databricks.com/en/sql/language-manual/functions/slashsign.html"], | ||
| }, | ||
| duckdb: { | ||
| integerDivisionTruncates: false, | ||
| sources: ["https://duckdb.org/docs/sql/functions/numeric"], | ||
| }, | ||
| mysql: { | ||
| integerDivisionTruncates: false, | ||
| sources: ["https://dev.mysql.com/doc/refman/8.0/en/arithmetic-functions.html"], | ||
| }, | ||
| oracle: { | ||
| integerDivisionTruncates: false, | ||
| sources: ["https://docs.oracle.com/en/database/oracle/oracle-database/19/sqlrf/Arithmetic-Operators.html"], | ||
| }, | ||
| clickhouse: { | ||
| integerDivisionTruncates: false, | ||
| sources: ["https://clickhouse.com/docs/en/sql-reference/functions/arithmetic-functions"], | ||
| }, | ||
| } | ||
|
|
||
| export function capabilityFor(dialect: string | undefined): AdapterCapability | undefined { | ||
| return dialect ? ADAPTER_CAPABILITIES[dialect.toLowerCase()] : undefined | ||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. P3: Prompt for AI agents
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Follow-up: nit, tracked ( |
||
| } | ||
|
|
||
| /** Display names of the engines where `int / int` truncates. */ | ||
| export const TRUNCATING_DIVISION_ENGINES = "Postgres, Redshift, Trino/Athena and SQL Server" | ||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. SUGGESTION: Unused export duplicates the engine list
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Follow-up: nit, tracked. Same as the cubic engine-list comment. |
||
| Original file line number | Diff line number | Diff line change | ||||
|---|---|---|---|---|---|---|
|
|
@@ -35,6 +35,21 @@ export const ReviewConfig = z.object({ | |||||
| * Functions missing on the project's own engine are always reported. | ||||||
| */ | ||||||
| portability: z.boolean().optional(), | ||||||
| /** | ||||||
| * Opt-in: report a new column missing from an enforced contract. `dbt build` | ||||||
| * already fails on it, so this only helps CI that runs `dbt parse`/`compile` | ||||||
| * without building. | ||||||
| */ | ||||||
| checkContractColumns: z.boolean().default(false), | ||||||
| /** | ||||||
| * Opt-in: explicit column lists outside dbt (loaders, exports) that must | ||||||
| * track a model's columns. `files` is a repo-relative glob, `pattern` a regex | ||||||
| * whose first capture group is the list, `model` the model it mirrors | ||||||
| * (omitted = files that mention the model by name). | ||||||
| */ | ||||||
| consumerColumnLists: z | ||||||
| .array(z.object({ files: z.string(), pattern: z.string(), model: z.string().optional() })) | ||||||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. P2: An invalid Prompt for AI agents
Suggested change
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Not changing: a bad consumer-list pattern is skipped with a warning on purpose, so an optional list never fails the whole review. Validating it in the schema is a possible follow-up. |
||||||
| .default([]), | ||||||
| /** Allow `--force-tier` (a debug bypass of the risk classifier). Set false | ||||||
| * in gate CI so a run cannot skip checks by forcing a lower tier. */ | ||||||
| allowForceTier: z.boolean().default(true), | ||||||
|
|
||||||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,106 @@ | ||
| import { execFile } from "node:child_process" | ||
| import { promisify } from "node:util" | ||
|
|
||
| /** | ||
| * Opt-in: explicit column lists outside dbt (loaders, exports, reverse-ETL | ||
| * configs) that must stay in step with a model's columns. A project names the | ||
| * files and a regex whose first capture group holds the list in | ||
| * `.altimate/review.yml`; the review then reports model columns a list is | ||
| * missing, and list entries the model no longer selects. Off unless configured, | ||
| * since no such convention is universal. | ||
| */ | ||
|
|
||
| export interface ConsumerColumnListConfig { | ||
| /** Repository-relative glob of the files holding the lists. */ | ||
| files: string | ||
| /** Regex whose first capture group is the column list. */ | ||
| pattern: string | ||
| /** The model the lists mirror; omitted = files that mention the model by name. */ | ||
| model?: string | ||
| } | ||
|
|
||
| export interface ConsumerColumnList { | ||
| path: string | ||
| model?: string | ||
| columns: string[] | ||
| /** File text, to match lists to models by name when `model` is omitted. */ | ||
| content: string | ||
| } | ||
|
|
||
| const exec = promisify(execFile) | ||
| const IDENT = /^[A-Za-z_][A-Za-z0-9_$]*$/ | ||
|
|
||
| /** One column list per match of `pattern` in `content` (each lowercased and | ||
| * de-duplicated), so two lists in one file are checked separately. */ | ||
| export function extractColumnLists(content: string, pattern: RegExp): string[][] { | ||
| const flags = pattern.flags.includes("g") ? pattern.flags : pattern.flags + "g" | ||
| const re = new RegExp(pattern.source, flags) | ||
| const lists: string[][] = [] | ||
| for (const m of content.matchAll(re)) { | ||
| const out = new Set<string>() | ||
| for (const token of (m[1] ?? "").split(/[,\s]+/)) { | ||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. P2: Whitespace splitting turns trailing comment words into column names, so Prompt for AI agents
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Follow-up: low priority, tracked. The list parser will split on commas, strip comments and keep quoted and dotted names. |
||
| const name = token.replace(/^["'`]+|["'`]+$/g, "") | ||
| if (IDENT.test(name)) out.add(name.toLowerCase()) | ||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. P2: Quoted column names containing characters outside Prompt for AI agents
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Follow-up: low priority, tracked. Same parser change as above. |
||
| } | ||
| if (out.size) lists.push([...out]) | ||
| } | ||
| return lists | ||
| } | ||
|
|
||
| /** Column names from every match of `pattern` in `content` (lowercased, de-duplicated). */ | ||
| export function extractColumnList(content: string, pattern: RegExp): string[] { | ||
| return [...new Set(extractColumnLists(content, pattern).flat())] | ||
| } | ||
|
|
||
| /** True when a list belongs to `model`: named explicitly, or the file mentions the model. */ | ||
| export function listMirrorsModel(list: ConsumerColumnList, model: string): boolean { | ||
| if (list.model) return list.model.toLowerCase() === model.toLowerCase() | ||
| return new RegExp(`\\b${model.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")}\\b`, "i").test(list.content) | ||
| } | ||
|
|
||
| /** | ||
| * Read the configured lists at the head (`head` ref, or the working tree). | ||
| * An invalid pattern is reported on stderr and skipped, never fatal. | ||
| */ | ||
| export async function loadConsumerLists(opts: { | ||
| configs: ConsumerColumnListConfig[] | ||
| gitRoot: string | ||
| head?: string | ||
| read: (repoRelPath: string) => Promise<string | undefined> | ||
| }): Promise<ConsumerColumnList[]> { | ||
| const out: ConsumerColumnList[] = [] | ||
| for (const cfg of opts.configs) { | ||
| let re: RegExp | ||
| try { | ||
| re = new RegExp(cfg.pattern, "g") | ||
| } catch (e) { | ||
| process.stderr.write(`⚠️ consumerColumnLists: invalid pattern ${JSON.stringify(cfg.pattern)} (${String(e)}); skipped.\n`) | ||
| continue | ||
| } | ||
| if (!/\((?!\?)/.test(cfg.pattern.replace(/\\./g, ""))) | ||
| process.stderr.write( | ||
| `⚠️ consumerColumnLists: pattern ${JSON.stringify(cfg.pattern)} has no capture group; it matches no list.\n`, | ||
| ) | ||
| // `git ls-tree` rejects `:(glob)` pathspec magic, so list every file and | ||
| // match the glob here, the same way for the head ref and the working tree. | ||
| const args = opts.head ? ["ls-tree", "-r", "-z", "--name-only", opts.head] : ["ls-files", "-z"] | ||
| const glob = new Bun.Glob(cfg.files) | ||
| let files: string[] = [] | ||
| try { | ||
| files = (await exec("git", args, { cwd: opts.gitRoot, maxBuffer: 64 * 1024 * 1024 })).stdout | ||
| .split("\0") | ||
| .filter((f) => f && glob.match(f)) | ||
| } catch (e) { | ||
| process.stderr.write( | ||
| `⚠️ consumerColumnLists: could not list files for ${JSON.stringify(cfg.files)} (${String(e).split("\n")[0]}); skipped.\n`, | ||
| ) | ||
| files = [] | ||
| } | ||
| for (const file of files) { | ||
| const content = await opts.read(file) | ||
| if (!content) continue | ||
| for (const columns of extractColumnLists(content, re)) out.push({ path: file, model: cfg.model, columns, content }) | ||
| } | ||
| } | ||
| return out | ||
| } | ||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
P3: This says every empty-input unit test is reported, but the rule only checks newly added or changed tests with inline
dictfixtures; unchanged and CSV/SQL-fixture tests are skipped. Document that scope to avoid promising findings the reviewer will not emit.Prompt for AI agents
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Follow-up: nit, tracked. The docs will narrow the unit-test scope.