Repository navigation
fix(review): verdict trust: only the engine blocks, policy from the base branch, allowForceTier #1722
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: fix/review-noise
Are you sure you want to change the base?
fix(review): verdict trust: only the engine blocks, policy from the base branch, allowForceTier #1722
Changes from all commits
9781a48
956f4ef
b354b9f
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -2,6 +2,7 @@ import z from "zod" | |
| import path from "node:path" | ||
| import { promises as fs } from "node:fs" | ||
| import YAML from "yaml" | ||
| import { createHash } from "node:crypto" | ||
| import { Rubric, DEFAULT_RUBRIC } from "./rubric" | ||
| import { ReviewMode } from "./verdict" | ||
| import { Severity } from "./finding" | ||
|
|
@@ -34,6 +35,9 @@ export const ReviewConfig = z.object({ | |
| * Functions missing on the project's own engine are always reported. | ||
| */ | ||
| portability: z.boolean().optional(), | ||
| /** Allow `--force-tier` (a debug bypass of the risk classifier). Set false | ||
| * in gate CI so a run cannot skip checks by forcing a lower tier. */ | ||
| allowForceTier: z.boolean().default(true), | ||
| /** | ||
| * Data-diff: actually run base-vs-head against the warehouse (core DataParity) | ||
| * and report row/value deltas. OPT-IN — it costs warehouse compute and needs a | ||
|
|
@@ -114,6 +118,106 @@ export function parseReviewConfig(text: string): ReviewConfig { | |
| return ReviewConfig.parse(data) | ||
| } | ||
|
|
||
| /** Where the applied policy came from. */ | ||
| export type PolicySource = "base" | "working_tree" | "default" | ||
|
|
||
| export interface ReviewPolicy { | ||
| config: ReviewConfig | ||
| source: PolicySource | ||
| /** Short hash of the applied config (canonical JSON), signed into the envelope. */ | ||
| hash: string | ||
| /** Settings the PR changes relative to the applied base policy (not applied). */ | ||
| changedInPr: string[] | ||
| } | ||
|
|
||
| function canonical(value: unknown): string { | ||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. SUGGESTION:
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Not changing: nit. They serve different modules; consolidation can happen in a cleanup pass. |
||
| if (value === null || typeof value !== "object") return JSON.stringify(value) ?? "null" | ||
| if (Array.isArray(value)) return "[" + value.map(canonical).join(",") + "]" | ||
| const obj = value as Record<string, unknown> | ||
| return ( | ||
| "{" + | ||
| Object.keys(obj) | ||
| .filter((k) => obj[k] !== undefined) | ||
| .sort() | ||
| .map((k) => JSON.stringify(k) + ":" + canonical(obj[k])) | ||
| .join(",") + | ||
| "}" | ||
| ) | ||
| } | ||
|
|
||
| /** Short, stable hash of a resolved config. */ | ||
| export function policyHash(config: ReviewConfig): string { | ||
| return createHash("sha256").update(canonical(config)).digest("hex").slice(0, 16) | ||
| } | ||
|
|
||
| /** Settings that differ between two configs: top-level keys, and `rubric.*` one level down. */ | ||
| export function policyChanges(base: ReviewConfig, head: ReviewConfig): string[] { | ||
| const out: string[] = [] | ||
| const keys = new Set([...Object.keys(base), ...Object.keys(head)]) as Set<keyof ReviewConfig> | ||
| for (const k of [...keys].sort()) { | ||
| if (canonical(base[k]) === canonical(head[k])) continue | ||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. SUGGESTION: Comparing parsed partial configs reports settings spelled out at their default values as changes Representation, not effective policy, drives this comparison. A base file with
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Follow-up: low priority, tracked. Compare |
||
| if (k === "rubric") { | ||
| const b = (base.rubric ?? {}) as Record<string, unknown> | ||
| const h = (head.rubric ?? {}) as Record<string, unknown> | ||
| for (const rk of [...new Set([...Object.keys(b), ...Object.keys(h)])].sort()) | ||
| if (canonical(b[rk]) !== canonical(h[rk])) out.push(`rubric.${rk}`) | ||
| } else out.push(k) | ||
| } | ||
| return out | ||
| } | ||
|
|
||
| async function readFirst(read: (rel: string) => Promise<string | undefined>): Promise<string | undefined> { | ||
| for (const rel of CANDIDATE_FILES) { | ||
| const text = await read(rel) | ||
| if (text !== undefined) return text | ||
| } | ||
| return undefined | ||
| } | ||
|
|
||
| /** | ||
| * Load the review policy the PR is judged by. | ||
| * | ||
| * With `readBase` (a reader over the merge-base commit), the policy is the | ||
| * BASE branch's `.altimate/review.yml`: a pull request must not be able to | ||
| * relax the rules it is reviewed under (switch `mode` to comment, drop a | ||
| * `blockOn` category, exclude its own paths, disable lanes). The PR's own | ||
| * version is parsed only to list what it would change, which the comment | ||
| * shows; it takes effect once merged. Without `readBase` (no git), the working | ||
| * tree's file applies. | ||
| */ | ||
| export async function loadReviewPolicy(opts: { | ||
| cwd: string | ||
| readBase?: (rel: string) => Promise<string | undefined> | ||
| }): Promise<ReviewPolicy> { | ||
| if (!opts.readBase) { | ||
| const config = await loadReviewConfig(opts.cwd) | ||
| const present = await readFirst((rel) => fs.readFile(path.join(opts.cwd, rel), "utf8").catch(() => undefined)) | ||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. SUGGESTION: The no-git branch reads the same candidate file twice
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Not changing: nit, one extra small file read on a non-default path. |
||
| return { config, source: present === undefined ? "default" : "working_tree", hash: policyHash(config), changedInPr: [] } | ||
| } | ||
| const baseText = await readFirst(opts.readBase) | ||
| let config: ReviewConfig | ||
| try { | ||
| config = baseText === undefined ? structuredClone(DEFAULT_REVIEW_CONFIG) : parseReviewConfig(baseText) | ||
| } catch (e: any) { | ||
| throw new Error( | ||
| `Failed to load the base branch's .altimate/review config: ${e?.message ?? e}. ` + | ||
| "The policy is read from the base branch so a PR cannot relax it; fix the file on the base branch " + | ||
| "(a PR that repairs it is reviewed under the broken base file, so merge it with an admin bypass).", | ||
| ) | ||
| } | ||
| const headText = await readFirst((rel) => fs.readFile(path.join(opts.cwd, rel), "utf8").catch(() => undefined)) | ||
|
cubic-dev-ai[bot] marked this conversation as resolved.
Resolved
|
||
| let changedInPr: string[] = [] | ||
| if (headText !== baseText) { | ||
| try { | ||
| const head = headText === undefined ? structuredClone(DEFAULT_REVIEW_CONFIG) : parseReviewConfig(headText) | ||
| changedInPr = policyChanges(config, head) | ||
| } catch { | ||
| changedInPr = ["(the PR's .altimate/review file does not parse)"] | ||
| } | ||
| } | ||
| return { config, source: baseText === undefined ? "default" : "base", hash: policyHash(config), changedInPr } | ||
| } | ||
|
|
||
| /** | ||
| * Load `.altimate/review.{yml,yaml,json}` from a project directory. Returns the | ||
| * defaults when no config file is present. Throws on a malformed/invalid file | ||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -84,9 +84,43 @@ export const Finding = z.object({ | |
| /** True when produced without a manifest/warehouse (lint-only degraded run). */ | ||
| degraded: z.boolean().default(false), | ||
| evidence: Evidence.optional(), | ||
| /** Which layer produced it: the deterministic engine, the regex/pattern | ||
| * catalog, or the advisory AI lane. Only `engine` findings decide the | ||
| * verdict unless the rubric opts a named catalog rule in. Stamped by | ||
| * buildEnvelope from `evidence.tool`. */ | ||
| origin: z.enum(["engine", "catalog", "ai"]).optional(), | ||
| }) | ||
| export type Finding = z.infer<typeof Finding> | ||
|
|
||
| export type FindingOrigin = NonNullable<Finding["origin"]> | ||
|
|
||
| /** Evidence tools backed by the deterministic engine (AST, manifest, lineage, | ||
| * warehouse) rather than text patterns. */ | ||
| const ENGINE_TOOLS = new Set(["impact_analysis", "dbt.manifest", "data.diff", "schema.detect_pii"]) | ||
|
|
||
| /** The layer a finding came from, always derived from `evidence.tool`: a | ||
| * supplied `origin` is ignored, so an externally built finding cannot claim | ||
| * to be engine-backed. Unknown or missing evidence counts as `catalog`, so a | ||
| * new detector can never block until it is classified. */ | ||
| export function findingOrigin(f: Pick<Finding, "evidence" | "origin">): FindingOrigin { | ||
| const tool = f.evidence?.tool ?? "" | ||
| if (tool === "ai-review") return "ai" | ||
| if (tool.startsWith("altimate_core.") || ENGINE_TOOLS.has(tool)) return "engine" | ||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. P1: The namespace prefix is too broad for the trust boundary: an unrecognized Prompt for AI agents
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Follow-up: low priority, tracked. Switch to an explicit allowlist of engine tools. |
||
| return "catalog" | ||
| } | ||
|
|
||
| /** A rule id in one spelling: catalog sources use `hardcoded-credential` and | ||
| * `pii_into_mart` alike, so ids compare lowercased with `_` as `-`. */ | ||
| export function normalizeRuleId(id: string): string { | ||
| return id.trim().toLowerCase().replace(/_/g, "-") | ||
| } | ||
|
|
||
| /** The catalog rule id behind a finding (`evidence.result.rule`), if any. */ | ||
| export function catalogRuleId(f: Pick<Finding, "evidence">): string | undefined { | ||
| const rule = (f.evidence?.result as { rule?: unknown } | undefined)?.rule | ||
| return typeof rule === "string" ? rule : undefined | ||
| } | ||
|
|
||
| /** Fields that define finding identity for fingerprinting / dedup. */ | ||
| export interface FingerprintInput { | ||
| category: ReviewCategory | string | ||
|
|
||
| Original file line number | Diff line number | Diff line change | ||||
|---|---|---|---|---|---|---|
|
|
@@ -226,6 +226,12 @@ export function coverageNotes(env: VerdictEnvelope): string[] { | |||||
| `${env.scope.movedUnchangedFiles} file(s) moved without edits; their SQL was not re-reviewed. ` + | ||||||
| "Folder-level config in `dbt_project.yml` (`+schema`, `+materialized`, `+tags`) can still change where and how a moved model builds.", | ||||||
| ) | ||||||
| if (env.policy?.changedInPr?.length) { | ||||||
| notes.push( | ||||||
| `This PR changes the review policy (\`.altimate/review.yml\`: ${env.policy.changedInPr.join(", ")}). ` + | ||||||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. P2: This note misidentifies the policy file when the repository uses the supported Prompt for AI agents
Suggested change
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Follow-up: low priority, tracked. Say "review policy" generically. |
||||||
| "The base branch's policy was applied; the change takes effect after merge.", | ||||||
| ) | ||||||
| } | ||||||
| if (env.ai && env.ai.state !== "ok") { | ||||||
| const what = { skipped: "skipped", timeout: "timed out", error: "failed" }[env.ai.state] | ||||||
| notes.push(`Advisory AI review ${what}${env.ai.reason ? `: ${env.ai.reason}` : ""}.`) | ||||||
|
|
||||||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -278,6 +278,21 @@ export async function resolveReviewBase(opts: ResolveBaseOptions): Promise<Resol | |
| return { ref: parent, sha, source: "parent" } | ||
| } | ||
|
|
||
| /** File content at a commit (`git show <ref>:<path>`), or undefined when the | ||
| * path does not exist there. `rel` is repository-relative, POSIX. */ | ||
| export async function showAtRef(cwd: string, ref: string, rel: string): Promise<string | undefined> { | ||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. WARNING: A failed
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Fixed in 956f4ef (see the cubic thread on |
||
| try { | ||
| return await git(["show", `${ref}:${rel}`], cwd) | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. [MINOR] return await git(["show", `${ref}:${rel}`], cwd)A failure other than a missing path (missing object in a partial clone, permissions) silently yields default policy ( Fix: return undefined only for git's "does not exist in" / "exists on disk, but not in" errors; rethrow otherwise. (Claude)
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Thanks. Fixed in |
||
| } catch (err) { | ||
| // Only "the path is not in that commit" means absent. Any other failure | ||
| // (bad ref, corrupt object, git missing) must surface: callers such as the | ||
| // base-policy loader would otherwise fall back to the permissive default. | ||
| const msg = err instanceof Error ? err.message : String(err) | ||
| if (/does not exist in|exists on disk, but not in/i.test(msg)) return undefined | ||
| throw err | ||
| } | ||
| } | ||
|
|
||
| /** Compute a short hash of the manifest file for the verdict envelope. */ | ||
| export async function manifestHash(manifestPath: string, cwd: string): Promise<string | undefined> { | ||
| try { | ||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,15 +1,16 @@ | ||
| import path from "node:path" | ||
| import { readFile, realpath, stat, access } from "node:fs/promises" | ||
| import { Installation } from "../../installation" | ||
| import { loadReviewConfig, resolveRubric } from "./config" | ||
| import type { Severity } from "./finding" | ||
| import { loadReviewPolicy, policyHash, resolveRubric } from "./config" | ||
| import { SEVERITY_ORDER, type Severity } from "./finding" | ||
| import { | ||
| collectChangedFiles, | ||
| makeContentResolver, | ||
| resolveReviewBase, | ||
| type ResolvedBase, | ||
| gitRepoRoot, | ||
| manifestHash, | ||
| showAtRef, | ||
| } from "./git" | ||
| import { makeCompiledResolver, dbtProjectName, dbtProjectLayout } from "./compiled" | ||
| import { scopeToProject } from "./diff-filter" | ||
|
|
@@ -217,19 +218,66 @@ async function detectStaleManifest(manifestAbs: string, changed: ChangedFile[], | |
| } | ||
|
|
||
| export async function reviewPullRequest(opts: ReviewPullRequestOptions): Promise<VerdictEnvelope> { | ||
| const config = await loadReviewConfig(opts.cwd) | ||
| if (opts.manifestPath) config.manifestPath = opts.manifestPath | ||
| if (opts.mode) config.mode = opts.mode | ||
| if (opts.severityThreshold) config.severityThreshold = opts.severityThreshold | ||
| const rubric = resolveRubric(config) | ||
|
|
||
| // Resolve the repo top-level once; used to root working-tree FS reads, the | ||
| // stale-manifest existence check, and the compiled-SQL resolver's path | ||
| // mapping. Falls back to opts.cwd when we couldn't resolve it (non-git or | ||
| // bare-repo contexts) — safe for the repo-root-invoked case but not helpful | ||
| // in the subdir-invocation case that this addresses. | ||
| const gitRoot = (await gitRepoRoot(opts.cwd)) ?? opts.cwd | ||
|
|
||
| // Only resolve a base ref if we actually need git (to collect changed files | ||
| // or to read old/new content). A caller that supplies BOTH `changedFiles` and | ||
| // `getContent` (e.g. a non-git CI integration) must not be forced through a | ||
| // git lookup that can fail when there's no usable history. | ||
| // | ||
| // The base is always the MERGE-BASE of the PR's target branch and the head: | ||
| // the diff, and every "old" file version, come from the commit the PR | ||
| // branched from, so commits the target gained since never leak in as changes. | ||
| const needGit = !opts.changedFiles || !opts.getContent | ||
|
cubic-dev-ai[bot] marked this conversation as resolved.
Resolved
|
||
| const resolved: ResolvedBase | undefined = needGit | ||
| ? await resolveReviewBase({ cwd: opts.cwd, base: opts.base, head: opts.head }) | ||
| : undefined | ||
|
|
||
| // The policy (.altimate/review.yml) comes from the BASE commit, so the PR | ||
| // under review cannot relax its own rules; what it would change is listed. | ||
| const cwdRel = await repoRelative(gitRoot, opts.cwd, true) | ||
| const policy = await loadReviewPolicy({ | ||
| cwd: opts.cwd, | ||
| readBase: | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. [MINOR] Callers supplying readBase:
resolved && cwdRel !== undefinedWhen both are supplied, Fix: let such integrations supply a trusted
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Thanks. I'm not changing this in this PR. That path exists for integrations without git history, so there is no trusted base to read, and the CLI and the action always resolve git. It is documented in the PR description. A caller-supplied |
||
| resolved && cwdRel !== undefined | ||
| ? (rel) => showAtRef(opts.cwd, resolved.sha, path.posix.join(cwdRel, rel)) | ||
| : undefined, | ||
| }) | ||
| const config = policy.config | ||
| if (policy.changedInPr.length) { | ||
| process.stderr.write( | ||
| `ℹ️ this PR changes .altimate/review (${policy.changedInPr.join(", ")}); the base branch's policy was applied.\n`, | ||
| ) | ||
| } | ||
| if (opts.forceTier && config.allowForceTier === false) { | ||
| throw new Error("--force-tier is disabled by `allowForceTier: false` in the base branch's .altimate/review.yml") | ||
| } | ||
| if (opts.manifestPath) config.manifestPath = opts.manifestPath | ||
| // `--mode` / `--severity` come from the PR-editable workflow. Against a base | ||
| // policy they may only tighten it: they cannot turn `gate` into `comment` | ||
| // or hide severities the base policy surfaces. | ||
| const fromBase = policy.source === "base" | ||
| if (opts.mode) { | ||
| if (fromBase && config.mode === "gate" && opts.mode !== "gate") { | ||
| process.stderr.write(`ℹ️ --mode ${opts.mode} ignored: the base branch's policy sets \`mode: gate\`.\n`) | ||
| } else config.mode = opts.mode | ||
| } | ||
| if (opts.severityThreshold) { | ||
| if (fromBase && SEVERITY_ORDER[opts.severityThreshold] > SEVERITY_ORDER[config.severityThreshold]) { | ||
| process.stderr.write( | ||
| `ℹ️ --severity ${opts.severityThreshold} ignored: the base branch's policy surfaces ${config.severityThreshold} and above.\n`, | ||
| ) | ||
| } else config.severityThreshold = opts.severityThreshold | ||
| } | ||
| // The signed hash describes the policy actually applied, after the flags. | ||
| policy.hash = policyHash(config) | ||
| const rubric = resolveRubric(config) | ||
|
|
||
| // Resolve the manifest against the PROJECT being reviewed (cwd), not the | ||
| // binary's process.cwd() — otherwise a relative path silently misses when the | ||
| // CLI is invoked from elsewhere, degrading every review to lint-only. | ||
|
|
@@ -271,18 +319,6 @@ export async function reviewPullRequest(opts: ReviewPullRequestOptions): Promise | |
| const projectRootRel = await repoRelative(gitRoot, dbtRoot, project !== undefined || dbtRoot !== opts.cwd) | ||
| const layout = projectRootRel !== undefined ? await dbtProjectLayout(dbtRoot, projectRootRel) : undefined | ||
|
|
||
| // Only resolve a base ref if we actually need git (to collect changed files | ||
| // or to read old/new content). A caller that supplies BOTH `changedFiles` and | ||
| // `getContent` (e.g. a non-git CI integration) must not be forced through a | ||
| // git lookup that can fail when there's no usable history. | ||
| // | ||
| // The base is always the MERGE-BASE of the PR's target branch and the head: | ||
| // the diff, and every "old" file version, come from the commit the PR | ||
| // branched from, so commits the target gained since never leak in as changes. | ||
| const needGit = !opts.changedFiles || !opts.getContent | ||
| const resolved: ResolvedBase | undefined = needGit | ||
| ? await resolveReviewBase({ cwd: opts.cwd, base: opts.base, head: opts.head }) | ||
| : undefined | ||
| if (resolved && resolved.source !== "flag") { | ||
| process.stderr.write(`ℹ️ review base: ${resolved.ref} (${resolved.source}), merge-base ${resolved.sha.slice(0, 12)}\n`) | ||
| } | ||
|
|
@@ -396,6 +432,7 @@ export async function reviewPullRequest(opts: ReviewPullRequestOptions): Promise | |
| staleManifest, | ||
| layout, | ||
| suppressPortability: config.portability === undefined ? dialectKnown : !config.portability, | ||
| policy: { source: policy.source, hash: policy.hash, changedInPr: policy.changedInPr.length ? policy.changedInPr : undefined }, | ||
|
cubic-dev-ai[bot] marked this conversation as resolved.
Resolved
|
||
| scope: { | ||
| baseRef: resolved?.ref ?? opts.base, | ||
| baseSha: resolved?.sha, | ||
|
|
||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
SUGGESTION: The docs state a stronger trust boundary than the shipped path enforces
The policy file is base-scoped, but
github/review/action.ymlalways passes--mode/--severityfrom the workflow (args=(--mode "$IN_MODE" ...)), and onpull_requestevents that workflow comes from the PR. A PR can therefore still switch the gate to comment mode by editing.github/workflows/*.yml— the PR description notes this as out of scope, but this bullet reads as if no PR-side edit can relax the gate. Adding that caveat (and pointing at CODEOWNERS / branch protection on.github/) would keep the doc accurate.There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Fixed in 956f4ef. Workflow flags can no longer relax a base policy, and the docs state the remaining caveat (protect
.github/).