From cda33969bc9de1aab7a6edc5ac9b0a5866136bf1 Mon Sep 17 00:00:00 2001 From: RaghwendraSingh Date: Thu, 12 Nov 2026 17:06:47 +0530 Subject: [PATCH 1/4] fix(review): failures are never clean: engine errors reported, per-file isolation, PII abstention, gate dismissal - The dispatcher runner records every engine call that throws (method, first reason, count) instead of silently returning empty. They are signed into the envelope (`engineErrors`), named in the comment, counted in `review_run.engine_errors`, and an empty review with engine errors is COMMENT, never APPROVE. - A failure while preparing one file, or a lane throwing for one file, marks that file unverified with the reason; the rest of the PR is still reviewed. PR-level lanes and the core-ran check are guarded the same way. - `detectPii` reports `abstained` when core could not analyse the SQL (check did not run, or its PII parser reported a parse error); the model is listed as not fully verified instead of passing as PII-clean. - When a rerun no longer blocks, earlier blocking altimate reviews are dismissed so the gate does not stay closed. The GitHub client is injectable so dedup, dismissal and skip rules are tested without the network. Co-Authored-By: Claude Opus 5.5 --- docs/docs/usage/dbt-pr-review.md | 9 +- .../opencode/src/altimate/review/format.ts | 23 +- .../src/altimate/review/orchestrate.ts | 105 +++++--- .../src/altimate/review/post-github.ts | 43 +++- .../opencode/src/altimate/review/runner.ts | 67 +++-- .../opencode/src/altimate/review/telemetry.ts | 1 + .../opencode/src/altimate/review/verdict.ts | 12 +- .../opencode/src/altimate/telemetry/index.ts | 2 + packages/opencode/src/cli/cmd/review.ts | 3 +- .../altimate/review-failures-gate.test.ts | 237 ++++++++++++++++++ 10 files changed, 438 insertions(+), 64 deletions(-) create mode 100644 packages/opencode/test/altimate/review-failures-gate.test.ts diff --git a/docs/docs/usage/dbt-pr-review.md b/docs/docs/usage/dbt-pr-review.md index c1fc58c71a..3da532522a 100644 --- a/docs/docs/usage/dbt-pr-review.md +++ b/docs/docs/usage/dbt-pr-review.md @@ -320,7 +320,14 @@ rubric: - **PII:** flagged only for PII columns the change newly exposes, not for columns the base already surfaced. That comparison needs the base version's SQL; when it can't be compiled, PII columns are reported as non-blocking - warnings that may predate the PR. + warnings that may predate the PR. A model the PII check could not analyse is + listed as not fully verified. +- **Failures are never a clean pass:** if an engine check throws, the comment + names it and the verdict is at most COMMENT, never APPROVE (envelope + `engineErrors`). A failure on one file marks that file unverified and the rest + of the PR is still reviewed. +- **The gate clears itself:** when a rerun no longer blocks, the earlier blocking + altimate review is dismissed (needs `pull-requests: write`). ### Risk-tier path tokens diff --git a/packages/opencode/src/altimate/review/format.ts b/packages/opencode/src/altimate/review/format.ts index 1b8f6be575..c033c81fb0 100644 --- a/packages/opencode/src/altimate/review/format.ts +++ b/packages/opencode/src/altimate/review/format.ts @@ -208,16 +208,19 @@ export function coverageNotes(env: VerdictEnvelope): string[] { const undecided = env.findings.filter((f) => f.degraded).length if (undecided) notes.push(`${undecided} finding(s) could not be decided by the engine and are marked _unverified_.`) const unverified = env.scope?.unverified ?? [] - if (unverified.length) { - // One line per reason: models can lack the head compile or the base compile. - const byReason = new Map() - for (const u of unverified) byReason.set(u.reason, [...(byReason.get(u.reason) ?? []), u.path]) - for (const [reason, paths] of byReason) { - const head = paths.slice(0, 5).map((p) => `\`${p}\``).join(", ") - const more = paths.length > 5 ? ` and ${paths.length - 5} more` : "" - notes.push(`${paths.length} model(s) not verified by the engine — ${reason}: ${head}${more}.`) - } - notes.push("Run `dbt compile` (and compile the base into `target-base/`) for equivalence and lineage checks.") + const byReason = new Map() + for (const u of unverified) byReason.set(u.reason, [...(byReason.get(u.reason) ?? []), u.path]) + for (const [reason, paths] of byReason) { + const head = paths.slice(0, 5).map((p) => `\`${p}\``).join(", ") + const more = paths.length > 5 ? ` and ${paths.length - 5} more` : "" + const hint = /compiled SQL/.test(reason) + ? " Run `dbt compile` (and compile the base into `target-base/`) for equivalence and lineage checks." + : "" + notes.push(`${paths.length} model(s) not fully verified — ${reason}: ${head}${more}.${hint}`) + } + if (env.engineErrors?.length) { + const list = env.engineErrors.map((e) => `${e.method} ×${e.count} (${e.reason})`).join("; ") + notes.push(`Some engine checks failed and did not run: ${list}. This verdict is not a clean approval.`) } if (env.scope?.outOfScopeFiles) notes.push(`${env.scope.outOfScopeFiles} changed file(s) outside the dbt project were not reviewed.`) diff --git a/packages/opencode/src/altimate/review/orchestrate.ts b/packages/opencode/src/altimate/review/orchestrate.ts index 9edfc1b22f..b3df1e2571 100644 --- a/packages/opencode/src/altimate/review/orchestrate.ts +++ b/packages/opencode/src/altimate/review/orchestrate.ts @@ -101,6 +101,8 @@ export interface GradeResult { export interface CheckResult { issues?: Array<{ rule: string; message: string; line?: number; severity?: string; category?: string }> piiColumns?: string[] + /** Core's PII pass could not parse the SQL: `piiColumns` is unknown, not empty. */ + piiAbstained?: boolean /** * True when the native core lint actually ran (parsed the SQL via the AST), * even if it found zero issues. Distinguishes "clean SQL" from "core @@ -118,7 +120,10 @@ export interface ReviewRunner { grade(sql: string, dialect: string): Promise check(sql: string, dialect: string, baseSql?: string): Promise equivalence(oldSql: string, newSql: string, dialect?: string): Promise - detectPii(sql: string, dialect: string): Promise<{ columns: string[] }> + /** `abstained`: the PII pass could not analyse the SQL, so `columns` is unknown. */ + detectPii(sql: string, dialect: string): Promise<{ columns: string[]; abstained?: boolean }> + /** Engine calls that threw during this review (method, first reason, count). */ + failures?(): Array<{ method: string; reason: string; count: number }> /** * Lexical scan (reserved-word aliases + dialect operators) — the curated lists * and detection live in the compiled core; this passes raw added diff lines. @@ -1156,35 +1161,55 @@ export async function runReview(input: OrchestrateInput): Promise false) } + // PR-level lanes (across files) that threw: reported with the engine errors. + const laneErrors: Array<{ method: string; reason: string; count: number }> = [] + const prLevel = (method: string, lane: Promise) => + lane.catch((err) => { + laneErrors.push({ method, reason: (err instanceof Error ? err.message : String(err)).slice(0, 200), count: 1 }) + return [] as Finding[] + }) + const failedFile = (file: { path: string }, err: unknown) => + unverified.push({ + path: file.path, + reason: `review failed for this file: ${(err instanceof Error ? err.message : String(err)).replace(/\s+/g, " ").slice(0, 160)}`, + }) await Promise.all( modelFiles.map(async (file) => { - const model = modelNameFromPath(file.path) - // For renames the previous content lives at oldPath, not the new path. - const oldRef = file.oldPath ?? file.path - const withOld = hasOldSide(file) - const [newSql, oldSql, compiledNew, compiledOld] = await Promise.all([ - file.status !== "deleted" ? getContent?.(file.path, "new") : Promise.resolve(undefined), - withOld ? getContent?.(oldRef, "old") : Promise.resolve(undefined), - file.status !== "deleted" ? getCompiled?.(file.path, "new") : Promise.resolve(undefined), - withOld ? getCompiled?.(oldRef, "old") : Promise.resolve(undefined), - ]) - // Engine lanes need dbt-compiled SQL. Raw SQL is used only when it holds - // no Jinja: the AST engine reads `{{ config() }}` as an injection attempt - // and a mixed raw-vs-compiled pair can never be proven equivalent, so a - // Jinja model without compiled output is marked unverified instead of - // producing warnings about its template syntax. The dbt-patterns lane - // always uses raw (it needs the Jinja). - const engineNewSql = compiledNew ?? (file.kind === "model_sql" && hasJinja(newSql) ? undefined : newSql) - const engineOldSql = compiledOld ?? (file.kind === "model_sql" && hasJinja(oldSql) ? undefined : oldSql) - if (file.status !== "deleted" && newSql && !engineNewSql) - unverified.push({ path: file.path, reason: "no compiled SQL for the head (run dbt compile)" }) - else if (withOld && oldSql && !engineOldSql) - unverified.push({ path: file.path, reason: "no compiled SQL for the base (compile the base into target-base)" }) - const impact = await input.runner.impact(model) - if (impact.hasManifest) anyManifest = true - const pii = engineNewSql ? (await input.runner.detectPii(engineNewSql, dialect)).columns : [] - const complex = engineNewSql && input.runner.isComplex ? await input.runner.isComplex(engineNewSql) : false - ctxByPath.set(file.path, { file, impact, pii, newSql, oldSql, engineNewSql, engineOldSql, complex }) + try { + const model = modelNameFromPath(file.path) + // For renames the previous content lives at oldPath, not the new path. + const oldRef = file.oldPath ?? file.path + const withOld = hasOldSide(file) + const [newSql, oldSql, compiledNew, compiledOld] = await Promise.all([ + file.status !== "deleted" ? getContent?.(file.path, "new") : Promise.resolve(undefined), + withOld ? getContent?.(oldRef, "old") : Promise.resolve(undefined), + file.status !== "deleted" ? getCompiled?.(file.path, "new") : Promise.resolve(undefined), + withOld ? getCompiled?.(oldRef, "old") : Promise.resolve(undefined), + ]) + // Engine lanes need dbt-compiled SQL. Raw SQL is used only when it holds + // no Jinja: the AST engine reads `{{ config() }}` as an injection attempt + // and a mixed raw-vs-compiled pair can never be proven equivalent, so a + // Jinja model without compiled output is marked unverified instead of + // producing warnings about its template syntax. The dbt-patterns lane + // always uses raw (it needs the Jinja). + const engineNewSql = compiledNew ?? (file.kind === "model_sql" && hasJinja(newSql) ? undefined : newSql) + const engineOldSql = compiledOld ?? (file.kind === "model_sql" && hasJinja(oldSql) ? undefined : oldSql) + if (file.status !== "deleted" && newSql && !engineNewSql) + unverified.push({ path: file.path, reason: "no compiled SQL for the head (run dbt compile)" }) + else if (withOld && oldSql && !engineOldSql) + unverified.push({ path: file.path, reason: "no compiled SQL for the base (compile the base into target-base)" }) + const impact = await input.runner.impact(model) + if (impact.hasManifest) anyManifest = true + const piiRes = engineNewSql ? await input.runner.detectPii(engineNewSql, dialect) : { columns: [] } + if (piiRes.abstained) unverified.push({ path: file.path, reason: "the PII check could not analyse this model" }) + const pii = piiRes.columns + const complex = engineNewSql && input.runner.isComplex ? await input.runner.isComplex(engineNewSql) : false + ctxByPath.set(file.path, { file, impact, pii, newSql, oldSql, engineNewSql, engineOldSql, complex }) + } catch (err) { + // One file's failure (unreadable content, an engine crash) degrades that + // file only; it must not abort the whole review. + failedFile(file, err) + } }), ) @@ -1337,20 +1362,32 @@ export async function runReview(input: OrchestrateInput): Promise + t.catch((err) => { + if (!laneFailed) failedFile(ctx.file, err) + laneFailed = true + return [] as Finding[] + }), + ), + ) + all.push(...settled) } // PR-level: cross-model sibling filter-consistency (divergent filters on a // shared upstream across the changed models). if (lanes.has("semantic_change")) { - all.push(await siblingConsistencyLane([...ctxByPath.values()], input.runner)) + all.push(await prLevel("sibling_consistency", siblingConsistencyLane([...ctxByPath.values()], input.runner))) } // PR-level: topology-aware equivalence for merge refactors (N models deleted + // 1 added that supersedes them). Per-model equivalence can't see these because // the new model has no 1:1 base. Advisory only (capped at warning). if (lanes.has("semantic_change")) { - all.push(await topologyEquivalenceLane(modelFiles, input.runner, getCompiled, dialect)) + all.push(await prLevel("topology_equivalence", topologyEquivalenceLane(modelFiles, input.runner, getCompiled, dialect))) } // schema.yml-level detectors (test removal) — run on changed YAML files @@ -1403,7 +1440,9 @@ export async function runReview(input: OrchestrateInput): Promise { if (!ctx.engineNewSql || ctx.file.status === "deleted") return // Cache hit: qualityLane already ran this exact check. - const cr = await input.runner.check(ctx.engineNewSql, dialect, ctx.engineOldSql) + const cr = await input.runner + .check(ctx.engineNewSql, dialect, ctx.engineOldSql) + .catch(() => ({ ran: false }) as CheckResult) if (cr.ran) coreRanFiles.add(ctx.file.path) }), ) @@ -1546,7 +1585,9 @@ export async function runReview(input: OrchestrateInput): Promise { - const octo = new Octokit({ auth: target.token }) +/** The Octokit surface the poster uses; injectable so the posting rules + * (dedup, dismissal, skip) are testable without the network. */ +export type GitHubClient = Pick & { + rest: Pick +} + +export async function postGitHubReview( + env: VerdictEnvelope, + target: GitHubTarget, + client?: GitHubClient, +): Promise { + const octo = client ?? new Octokit({ auth: target.token }) const { owner, repo, prNumber } = target const result: PostResult = { inlineFellBack: false } @@ -99,13 +111,16 @@ export async function postGitHubReview(env: VerdictEnvelope, target: GitHubTarge // ignores line numbers) is not posted again on the next push. let posted = new Set() let priorBotReview = false + let staleBlocking: number[] = [] try { const [reviewComments, reviews] = await Promise.all([ octo.paginate(octo.rest.pulls.listReviewComments, { owner, repo, pull_number: prNumber, per_page: 100 }), octo.paginate(octo.rest.pulls.listReviews, { owner, repo, pull_number: prNumber, per_page: 100 }), ]) posted = postedFindingIds(reviewComments.map((c) => c.body)) - priorBotReview = reviews.some((r) => BOT_REVIEW_RE.test(r.body ?? "")) + const botReviews = reviews.filter((r) => BOT_REVIEW_RE.test(r.body ?? "")) + priorBotReview = botReviews.length > 0 + staleBlocking = botReviews.filter((r) => r.state === "CHANGES_REQUESTED").map((r) => r.id) } catch { /* listing failed: post everything rather than nothing */ } @@ -113,6 +128,28 @@ export async function postGitHubReview(env: VerdictEnvelope, target: GitHubTarge const withLine = env.findings.filter((f) => typeof f.startLine === "number").length result.inlineSkippedAsPosted = withLine - comments.length + // Gate lifecycle: when this run no longer blocks, an earlier blocking + // altimate review would keep the PR blocked forever. Dismiss it, pointing at + // the new verdict. (Requires `pull-requests: write`; a failure is non-fatal.) + if (event === "COMMENT" && staleBlocking.length) { + let dismissed = 0 + for (const id of staleBlocking) { + try { + await octo.rest.pulls.dismissReview({ + owner, + repo, + pull_number: prNumber, + review_id: id, + message: `Superseded by a newer altimate review: ${reviewBody}`, + }) + dismissed++ + } catch { + /* keep going; the summary still shows the current verdict */ + } + } + result.dismissedReviews = dismissed + } + // Nothing new inline, not blocking, and the PR already has an altimate // review: the updated summary comment says it all; another "Reviewed with // comments" entry per push is noise. A blocking verdict always posts. diff --git a/packages/opencode/src/altimate/review/runner.ts b/packages/opencode/src/altimate/review/runner.ts index eec3732d81..aa59b398ca 100644 --- a/packages/opencode/src/altimate/review/runner.ts +++ b/packages/opencode/src/altimate/review/runner.ts @@ -102,6 +102,16 @@ export interface DispatcherRunnerOptions { export function createDispatcherRunner(opts: DispatcherRunnerOptions): ReviewRunner { const checkCache = new Map() + // Engine calls that threw, by method → first reason. Each method still + // returns its empty result so one failure never aborts the review, but the + // failures are reported (envelope `engineErrors`, the comment, telemetry) + // instead of reading as "no findings". + const failures = new Map() + function fail(method: string, err: unknown) { + const reason = (err instanceof Error ? err.message : String(err)).replace(/\s+/g, " ").slice(0, 200) + const prev = failures.get(method) + failures.set(method, { reason: prev?.reason ?? reason, count: (prev?.count ?? 0) + 1 }) + } let manifestPromise: Promise | undefined async function loadManifest(): Promise { @@ -252,8 +262,14 @@ export function createDispatcherRunner(opts: DispatcherRunnerOptions): ReviewRun ].filter((c): c is string => typeof c === "string" && !!c) // ran=true: the core parsed and analyzed the SQL (even if zero issues). // This lets the orchestrator defer structural checks to the AST lint. - out = { issues, piiColumns: [...new Set(piiColumns)], ran: true } - } catch { + out = { + issues, + piiColumns: [...new Set(piiColumns)], + ran: true, + piiAbstained: !!(data.pii as any)?.parse_error, + } + } catch (err) { + fail("check", err) out = { issues: [], piiColumns: [], ran: false } } checkCache.set(cacheKey, out) @@ -310,7 +326,8 @@ export function createDispatcherRunner(opts: DispatcherRunnerOptions): ReviewRun const res = await Dispatcher.call("altimate_core.grade", { sql, schema_context: await resolveSchema() }) const data = (res.data ?? {}) as Record return { grade: data.overall_grade ?? data.grade } - } catch { + } catch (err) { + fail("grade", err) return {} } }, @@ -324,7 +341,8 @@ export function createDispatcherRunner(opts: DispatcherRunnerOptions): ReviewRun const res = await Dispatcher.call("altimate_core.grain", { sql }) const d = (res.data ?? {}) as Record return { group_by: asArray(d.group_by), dedup_partition: asArray(d.dedup_partition) } - } catch { + } catch (err) { + fail("grain", err) return { group_by: [], dedup_partition: [] } } }, @@ -337,14 +355,16 @@ export function createDispatcherRunner(opts: DispatcherRunnerOptions): ReviewRun try { const res = await Dispatcher.call("altimate_core.dbt_config_lint", { sql: rawSql }) out.push(...asArray((res.data as any)?.findings)) - } catch { + } catch (err) { + fail("dbtConfigLint", err) /* skip */ } if (oldRawSql) { try { const res = await Dispatcher.call("altimate_core.dbt_config_diff", { base_sql: oldRawSql, head_sql: rawSql }) out.push(...asArray((res.data as any)?.findings)) - } catch { + } catch (err) { + fail("dbtConfigLint", err) /* skip */ } } @@ -358,7 +378,8 @@ export function createDispatcherRunner(opts: DispatcherRunnerOptions): ReviewRun try { const res = await Dispatcher.call("altimate_core.structural_diff", { base_sql: baseSql, head_sql: headSql }) return asArray((res.data as any)?.findings) - } catch { + } catch (err) { + fail("structuralDiff", err) return [] } }, @@ -368,7 +389,8 @@ export function createDispatcherRunner(opts: DispatcherRunnerOptions): ReviewRun const res = await Dispatcher.call("altimate_core.source_filters", { sql }) const d = (res.data ?? {}) as Record return (d.filters ?? {}) as Record - } catch { + } catch (err) { + fail("sourceFilters", err) return {} } }, @@ -397,7 +419,8 @@ export function createDispatcherRunner(opts: DispatcherRunnerOptions): ReviewRun const res = await Dispatcher.call("altimate_core.metadata", { sql }) const d = (res.data ?? {}) as Record return !!(d.has_window_functions || d.has_subqueries || (typeof d.node_count === "number" && d.node_count >= 12)) - } catch { + } catch (err) { + fail("isComplex", err) return false } }, @@ -423,7 +446,8 @@ export function createDispatcherRunner(opts: DispatcherRunnerOptions): ReviewRun confidence: typeof c.confidence === "number" ? c.confidence : 0, masking: c.suggested_masking ?? undefined, })) - } catch { + } catch (err) { + fail("classifyPii", err) return [] } }, @@ -450,7 +474,8 @@ export function createDispatcherRunner(opts: DispatcherRunnerOptions): ReviewRun rowsChanged: o.rows_changed ?? o.different ?? o.changed, summary: o.summary ?? r.summary, } - } catch { + } catch (err) { + fail("dataDiff", err) return null } }, @@ -468,7 +493,8 @@ export function createDispatcherRunner(opts: DispatcherRunnerOptions): ReviewRun source: String(e.source ?? ""), target: String(e.target ?? ""), })) - } catch { + } catch (err) { + fail("columnLineage", err) return [] } }, @@ -484,7 +510,8 @@ export function createDispatcherRunner(opts: DispatcherRunnerOptions): ReviewRun message: String(f.message ?? ""), line: f.line, })) - } catch { + } catch (err) { + fail("lexicalScan", err) return [] } }, @@ -529,16 +556,24 @@ export function createDispatcherRunner(opts: DispatcherRunnerOptions): ReviewRun differences: diffs.map((d) => d?.description ?? String(d)), confidence: EngineCoerce.bandConfidence(data.confidence), } - } catch { + } catch (err) { + fail("equivalence", err) return { decided: false } } }, - async detectPii(sql: string): Promise<{ columns: string[] }> { + async detectPii(sql: string): Promise<{ columns: string[]; abstained?: boolean }> { // Text-based PII comes from altimate_core.check (schema.detect_pii is // warehouse-based and unavailable in CI). Reuses the memoized check. + // When core could not analyse the SQL (check did not run, or its PII + // parser reported a parse error) the empty list means "unknown", not + // "no PII": say so. const result = await runCheck(sql) - return { columns: result.piiColumns ?? [] } + return { columns: result.piiColumns ?? [], abstained: !result.ran || result.piiAbstained === true } + }, + + failures() { + return [...failures].map(([method, f]) => ({ method, reason: f.reason, count: f.count })) }, } } diff --git a/packages/opencode/src/altimate/review/telemetry.ts b/packages/opencode/src/altimate/review/telemetry.ts index 3fbe4af1bb..0e6f4c0ad5 100644 --- a/packages/opencode/src/altimate/review/telemetry.ts +++ b/packages/opencode/src/altimate/review/telemetry.ts @@ -117,6 +117,7 @@ export function emitReviewRun(input: { ai_duration_ms: env.ai?.durationMs, base_source: env.scope?.baseSource, out_of_scope_files: env.scope?.outOfScopeFiles ?? 0, + engine_errors: env.engineErrors?.reduce((n, e) => n + e.count, 0) ?? 0, stale_manifest: env.staleManifest === true, critical: env.summary.critical, warning: env.summary.warning, diff --git a/packages/opencode/src/altimate/review/verdict.ts b/packages/opencode/src/altimate/review/verdict.ts index 5a45e14351..b30d5f5fc4 100644 --- a/packages/opencode/src/altimate/review/verdict.ts +++ b/packages/opencode/src/altimate/review/verdict.ts @@ -156,6 +156,11 @@ export const VerdictEnvelope = z.object({ staleManifest: z.boolean().optional(), /** Base resolution and review coverage (see ReviewScope). */ scope: ReviewScope.optional(), + /** Engine calls that failed during the review. Their checks did not run, so + * the verdict is never a clean APPROVE when this is present. */ + engineErrors: z + .array(z.object({ method: z.string(), reason: z.string(), count: z.number().int().positive() })) + .optional(), /** The review policy applied: from the base branch, the working tree, or * defaults; its hash; and settings the PR changes but that were not applied. */ policy: z @@ -235,6 +240,7 @@ export interface BuildEnvelopeInput { scope?: ReviewScope ai?: VerdictEnvelope["ai"] policy?: VerdictEnvelope["policy"] + engineErrors?: VerdictEnvelope["engineErrors"] } function summarize(findings: Finding[], degraded: boolean): VerdictEnvelope["summary"] { @@ -246,7 +252,10 @@ function summarize(findings: Finding[], degraded: boolean): VerdictEnvelope["sum /** Assemble the verdict envelope (unsigned). Call signEnvelope to sign it. */ export function buildEnvelope(input: BuildEnvelopeInput): VerdictEnvelope { const rubric = input.rubric ?? DEFAULT_RUBRIC - const ideal = computeIdealVerdict(input.findings, rubric) + // An engine failure means some checks did not run: "no findings" is then + // unknown, not clean, so the verdict cannot be APPROVE. + const computed = computeIdealVerdict(input.findings, rubric) + const ideal = computed === "APPROVE" && input.engineErrors?.length ? "COMMENT" : computed const verdict = applyMode(ideal, input.mode) const degraded = input.degraded ?? input.findings.some((f) => f.degraded) return VerdictEnvelope.parse({ @@ -266,6 +275,7 @@ export function buildEnvelope(input: BuildEnvelopeInput): VerdictEnvelope { scope: input.scope, ai: input.ai, policy: input.policy, + engineErrors: input.engineErrors, generatedAt: input.generatedAt, }) } diff --git a/packages/opencode/src/altimate/telemetry/index.ts b/packages/opencode/src/altimate/telemetry/index.ts index 8566507850..288b69319a 100644 --- a/packages/opencode/src/altimate/telemetry/index.ts +++ b/packages/opencode/src/altimate/telemetry/index.ts @@ -1608,6 +1608,8 @@ export namespace Telemetry { /** Where the review base came from (flag, github_event, default_branch, …). */ base_source?: string out_of_scope_files?: number + /** Engine calls that threw during the review (their checks did not run). */ + engine_errors?: number stale_manifest?: boolean critical?: number warning?: number diff --git a/packages/opencode/src/cli/cmd/review.ts b/packages/opencode/src/cli/cmd/review.ts index cef6942c20..b1677b7fe1 100644 --- a/packages/opencode/src/cli/cmd/review.ts +++ b/packages/opencode/src/cli/cmd/review.ts @@ -194,7 +194,8 @@ export const ReviewCommand = cmd({ `Posted review to ${where}` + (r.inlineFellBack ? " (inline comments fell back to summary-only)" : "") + (r.inlineSkippedAsPosted ? ` (${r.inlineSkippedAsPosted} inline already on the PR)` : "") + - (r.reviewSkipped ? " (summary updated; no new review needed)" : ""), + (r.reviewSkipped ? " (summary updated; no new review needed)" : "") + + (r.dismissedReviews ? ` (dismissed ${r.dismissedReviews} earlier blocking review(s))` : ""), ) } } diff --git a/packages/opencode/test/altimate/review-failures-gate.test.ts b/packages/opencode/test/altimate/review-failures-gate.test.ts new file mode 100644 index 0000000000..07f2c4e252 --- /dev/null +++ b/packages/opencode/test/altimate/review-failures-gate.test.ts @@ -0,0 +1,237 @@ +import { describe, test, expect } from "bun:test" +import { runReview, type ReviewRunner } from "../../src/altimate/review/orchestrate" +import { DEFAULT_REVIEW_CONFIG } from "../../src/altimate/review/config" +import { DEFAULT_RUBRIC } from "../../src/altimate/review/rubric" +import { makeFinding } from "../../src/altimate/review/finding" +import { buildEnvelope, signEnvelope, verifyEnvelope } from "../../src/altimate/review/verdict" +import { renderSummary, findingMarker } from "../../src/altimate/review/format" +import { postGitHubReview, type GitHubClient } from "../../src/altimate/review/post-github" + +/** + * A review must never look clean because something broke: engine failures and + * PII abstentions are reported, one file's crash degrades only that file, and + * a blocking review is cleared when a rerun no longer blocks. + */ + +function runner(over: Partial = {}): ReviewRunner { + return { + async impact() { + return { hasManifest: true, severity: "SAFE", directCount: 0, transitiveCount: 0, testCount: 0 } + }, + async grade() { + return { grade: "B" } + }, + async check() { + return { issues: [], ran: true } + }, + async equivalence() { + return { decided: true, equivalent: true } + }, + async detectPii() { + return { columns: [] } + }, + ...over, + } as ReviewRunner +} + +const two = [ + { path: "models/a.sql", status: "added" as const, diff: "+select 1 as id\n" }, + { path: "models/b.sql", status: "added" as const, diff: "+select 2 as id\n" }, +] + +describe("engine failures are never clean", () => { + test("no findings plus an engine failure is COMMENT, not APPROVE, and the failure is shown", async () => { + const env = await runReview({ + changedFiles: two, + config: { ...DEFAULT_REVIEW_CONFIG, reviewers: ["sql_quality"] }, + rubric: DEFAULT_RUBRIC, + mode: "comment", + runner: runner({ failures: () => [{ method: "check", reason: "native core not loaded", count: 2 }] }), + getContent: async (f) => (f.endsWith("a.sql") ? "select 1 as id" : "select 2 as id"), + }) + expect(env.findings).toHaveLength(0) + expect(env.idealVerdict).toBe("COMMENT") + expect(env.engineErrors).toEqual([{ method: "check", reason: "native core not loaded", count: 2 }]) + expect(renderSummary(env)).toContain("Some engine checks failed and did not run: check ×2 (native core not loaded)") + }) + + test("engine errors are signed", () => { + const env = signEnvelope( + buildEnvelope({ findings: [], tier: "lite", mode: "comment", engineErrors: [{ method: "equivalence", reason: "x", count: 1 }] }), + ) + expect(env.verdict).toBe("COMMENT") + expect(verifyEnvelope(env)).toBe(true) + expect(verifyEnvelope({ ...env, engineErrors: undefined })).toBe(false) + }) + + test("without failures an empty review still approves", () => { + expect(buildEnvelope({ findings: [], tier: "lite", mode: "comment" }).verdict).toBe("APPROVE") + }) +}) + +describe("one file's failure degrades only that file", () => { + test("unreadable content for one model: the other is still reviewed", async () => { + let checked: string[] = [] + const env = await runReview({ + changedFiles: two, + config: { ...DEFAULT_REVIEW_CONFIG, reviewers: ["sql_quality"] }, + rubric: DEFAULT_RUBRIC, + mode: "comment", + runner: runner({ + async check(sql: string) { + checked.push(sql) + return { issues: [], ran: true } + }, + }), + getContent: async (f) => { + if (f.endsWith("a.sql")) throw new Error("EACCES: permission denied") + return "select 2 as id" + }, + }) + expect(checked).toContain("select 2 as id") + expect(env.scope?.unverified).toEqual([ + { path: "models/a.sql", reason: "review failed for this file: EACCES: permission denied" }, + ]) + }) + + test("a lane that throws on one file keeps the other file's findings", async () => { + const env = await runReview({ + changedFiles: two, + config: { ...DEFAULT_REVIEW_CONFIG, reviewers: ["sql_quality"] }, + rubric: DEFAULT_RUBRIC, + mode: "comment", + runner: runner({ + async check(sql: string) { + if (sql.includes("1")) throw new Error("engine panic") + return { issues: [{ rule: "select-star", message: "avoid", severity: "warning" }], ran: true } + }, + }), + getContent: async (f) => (f.endsWith("a.sql") ? "select 1 as id" : "select 2 as id"), + }) + expect(env.findings.some((f) => f.file === "models/b.sql")).toBe(true) + expect(env.scope?.unverified?.map((u) => u.path)).toEqual(["models/a.sql"]) + }) +}) + +describe("PII abstention", () => { + test("a PII check that could not analyse the model is reported, not treated as clean", async () => { + const env = await runReview({ + changedFiles: [two[0]], + config: { ...DEFAULT_REVIEW_CONFIG, reviewers: ["sql_quality"] }, + rubric: DEFAULT_RUBRIC, + mode: "comment", + runner: runner({ + async detectPii() { + return { columns: [], abstained: true } + }, + }), + getContent: async () => "select 1 as id", + }) + expect(env.scope?.unverified).toEqual([{ path: "models/a.sql", reason: "the PII check could not analyse this model" }]) + expect(renderSummary(env)).toContain("the PII check could not analyse this model") + }) +}) + +describe("posting: dedup, dismissal of a stale block, no repeat reviews", () => { + function fakeClient(state: { + reviewComments?: Array<{ body: string }> + reviews?: Array<{ id: number; body: string; state: string }> + }) { + const calls: Record = { createReview: [], dismissReview: [], createComment: [], updateComment: [] } + const client = { + async paginate(fn: any) { + if (fn === client.rest.issues.listComments) return [] + if (fn === client.rest.pulls.listReviewComments) return state.reviewComments ?? [] + if (fn === client.rest.pulls.listReviews) return state.reviews ?? [] + return [] + }, + rest: { + issues: { + listComments: () => {}, + async createComment(a: any) { + calls.createComment.push(a) + return { data: { id: 1 } } + }, + async updateComment(a: any) { + calls.updateComment.push(a) + return { data: { id: 1 } } + }, + }, + pulls: { + listReviewComments: () => {}, + listReviews: () => {}, + async get() { + return { data: { head: { sha: "abc" } } } + }, + async createReview(a: any) { + calls.createReview.push(a) + return { data: { id: 99 } } + }, + async dismissReview(a: any) { + calls.dismissReview.push(a) + return { data: {} } + }, + }, + }, + } + return { client: client as unknown as GitHubClient, calls } + } + const target = { token: "t", owner: "o", repo: "r", prNumber: 7 } + const engineFinding = (line?: number) => + makeFinding({ + severity: "warning", + category: "sql_quality", + title: "m: select-star", + body: "avoid", + file: "models/m.sql", + model: "m", + startLine: line, + evidence: { tool: "altimate_core.check", result: {} }, + ruleKey: "q:select-star", + }) + + test("a clean rerun dismisses the earlier blocking altimate review", async () => { + const env = buildEnvelope({ findings: [], tier: "lite", mode: "gate" }) + const { client, calls } = fakeClient({ + reviews: [ + { id: 11, body: "🛑 Changes requested — 1 critical (full tier)", state: "CHANGES_REQUESTED" }, + { id: 12, body: "LGTM from a human", state: "CHANGES_REQUESTED" }, + ], + }) + const r = await postGitHubReview(env, target, client) + expect(calls.dismissReview.map((c) => c.review_id)).toEqual([11]) + expect(r.dismissedReviews).toBe(1) + }) + + test("a still-blocking rerun does not dismiss and posts REQUEST_CHANGES", async () => { + const blocking = makeFinding({ + severity: "critical", + category: "lineage_breakage", + title: "m: breaks downstream", + body: "x", + file: "models/m.sql", + evidence: { tool: "impact_analysis", result: {} }, + }) + const env = buildEnvelope({ findings: [blocking], tier: "full", mode: "gate" }) + const { client, calls } = fakeClient({ + reviews: [{ id: 11, body: "🛑 Changes requested — 1 critical (full tier)", state: "CHANGES_REQUESTED" }], + }) + await postGitHubReview(env, target, client) + expect(calls.dismissReview).toHaveLength(0) + expect(calls.createReview[0].event).toBe("REQUEST_CHANGES") + }) + + test("findings already on the PR are not re-posted, and with nothing new no review is added", async () => { + const f = engineFinding(3) + const env = buildEnvelope({ findings: [f], tier: "lite", mode: "comment" }) + const { client, calls } = fakeClient({ + reviewComments: [{ body: `⚠️ old\n\n${findingMarker(f.id)}` }], + reviews: [{ id: 5, body: "💬 Reviewed with comments — 1 warning (lite tier)", state: "COMMENTED" }], + }) + const r = await postGitHubReview(env, target, client) + expect(r.inlineSkippedAsPosted).toBe(1) + expect(r.reviewSkipped).toBe(true) + expect(calls.createReview).toHaveLength(0) + expect(calls.createComment.length + calls.updateComment.length).toBe(1) + }) +}) From 2f865225ec2f9cc811cc61dace9215eff3df0743 Mon Sep 17 00:00:00 2001 From: RaghwendraSingh Date: Fri, 13 Nov 2026 11:10:41 +0530 Subject: [PATCH 2/4] fix(review): a failed run never clears a block; shallow CI checkouts fetch the base; upgrade note for gate users - a stale `CHANGES_REQUESTED` review is dismissed only when the new run completed (no `engineErrors`, no file that failed); an engine crash no longer unblocks a PR by making its findings disappear - in CI, a missing target branch is fetched from `origin` and a shallow history deepened (50/200/1000) until the merge-base exists, so the default `actions/checkout` works without `fetch-depth: 0`; still an error when that fails - docs: what changes for existing `gate` setups (pattern rules no longer block unless listed in `rubric.blockingRules`) and the new checkout behavior Co-Authored-By: Claude Opus 5.5 --- docs/docs/usage/dbt-pr-review.md | 10 ++++- packages/opencode/src/altimate/review/git.ts | 38 +++++++++++++++++-- .../src/altimate/review/post-github.ts | 10 ++++- .../altimate/review-failures-gate.test.ts | 23 +++++++++++ .../test/altimate/review-scope-diff.test.ts | 33 ++++++++++++++++ 5 files changed, 108 insertions(+), 6 deletions(-) diff --git a/docs/docs/usage/dbt-pr-review.md b/docs/docs/usage/dbt-pr-review.md index 3da532522a..de60bbb378 100644 --- a/docs/docs/usage/dbt-pr-review.md +++ b/docs/docs/usage/dbt-pr-review.md @@ -108,7 +108,7 @@ Options: | Flag | Description | |------|-------------| -| `--base ` | Base git ref. Defaults to the pull request's target branch in CI (GitHub `pull_request.base.ref` or `GITHUB_BASE_REF`, GitLab `CI_MERGE_REQUEST_TARGET_BRANCH_NAME`), else the remote default branch. Always reduced to the merge-base with the head, so commits the base gained after the PR branched are never reviewed. In CI, a target branch that is not fetched is an error (use `fetch-depth: 0`). | +| `--base ` | Base git ref. Defaults to the pull request's target branch in CI (GitHub `pull_request.base.ref` or `GITHUB_BASE_REF`, GitLab `CI_MERGE_REQUEST_TARGET_BRANCH_NAME`), else the remote default branch. Always reduced to the merge-base with the head, so commits the base gained after the PR branched are never reviewed. In CI, a shallow checkout is handled: the target branch is fetched and the history deepened until the merge-base is found. If that fails (no `origin`, no access), the review stops with an error rather than diffing against another branch. | | `--head ` | Head git ref. Omit to review the working tree. | | `--manifest ` | Path to the compiled `manifest.json`. When omitted, the reviewer walks up from the current directory to find the nearest `dbt_project.yml` and uses its adjacent `target/manifest.json`; the discovered path is logged to stderr. | | `--mode comment\|gate` | `comment` never blocks; `gate` exits non-zero on `REQUEST_CHANGES`. | @@ -203,7 +203,7 @@ jobs: | Input | How to produce it | Unlocks | |---|---|---| -| Full git history of the PR's target branch | `actions/checkout` with `fetch-depth: 0` | Correct base: the diff starts at the merge-base with the PR's target branch. A target branch that is not fetched is an error, never a silent fallback to `main`. | +| The PR's target branch and the merge-base | Fetched automatically from `origin` when the checkout is shallow; `fetch-depth: 0` avoids the extra fetches | Correct base: the diff starts at the merge-base with the PR's target branch. When it can't be fetched, the review stops with an error, never a silent fallback to `main`. | | `target/manifest.json` | `dbt compile` (or `dbt parse` for lint + lineage only) | Blast radius, lineage breakage, tiering. Without it the run is lint-only. | | `target/compiled/…` | `dbt compile` | Engine checks on the head (lint, grade, PII, grain). | | `target-base/compiled/…` | Compile the base ref into `target-base/` (below) | Equivalence proofs and column-breakage for modified models. | @@ -303,6 +303,12 @@ rubric: rules) or `ai`. Only `engine` findings count toward REQUEST_CHANGES. A pattern rule you trust can be allowed to block by naming its id in `rubric.blockingRules`. The advisory AI lane never blocks. +- **Upgrading a `gate` setup.** Before this policy, any critical finding in a + `blockOn` category could block, including pattern rules such as the regex + PII check. Now pattern-rule findings comment but don't block. To keep a + pattern rule blocking, add its id to `rubric.blockingRules`. A blocking + review is dismissed only by a later run that completed: when an engine call + or a file failed, the earlier block stays. ### What the PR comment shows diff --git a/packages/opencode/src/altimate/review/git.ts b/packages/opencode/src/altimate/review/git.ts index d7026f82c5..055842f958 100644 --- a/packages/opencode/src/altimate/review/git.ts +++ b/packages/opencode/src/altimate/review/git.ts @@ -202,6 +202,33 @@ async function mergeBase(a: string, b: string, cwd: string): Promise Promise): Promise { + if (branch.startsWith("-")) return false + try { + await git(["fetch", "--no-tags", "origin", `+refs/heads/${branch}:refs/remotes/origin/${branch}`], cwd) + } catch { + return false + } + if (await found()) return true + for (const depth of [50, 200, 1000]) { + try { + if ((await git(["rev-parse", "--is-shallow-repository"], cwd)).trim() !== "true") return false + const headSha = (await git(["rev-parse", "HEAD"], cwd)).trim() + await git(["fetch", "--no-tags", `--deepen=${depth}`, "origin", `+refs/heads/${branch}:refs/remotes/origin/${branch}`, headSha], cwd) + } catch { + return false + } + if (await found()) return true + } + return false +} + /** The PR's target branch from the CI provider, when the run is for a PR/MR. */ async function ciTargetBranch(env: NodeJS.ProcessEnv): Promise<{ branch: string; source: BaseSource } | undefined> { if (env.GITHUB_EVENT_PATH) { @@ -241,10 +268,15 @@ export async function resolveReviewBase(opts: ResolveBaseOptions): Promise { + for (const ref of [`origin/${target.branch}`, target.branch]) { + const sha = await mergeBase(ref, head, opts.cwd) + if (sha) return { ref, sha, source: target.source } + } + return undefined } + const found = (await find()) ?? ((await fetchTargetHistory(target.branch, opts.cwd, find)) ? await find() : undefined) + if (found) return found throw new Error( `PR target branch '${target.branch}' (${target.source}) is not available locally. ` + `Fetch it (actions/checkout with fetch-depth: 0, or git fetch origin ${target.branch}) or pass --base.`, diff --git a/packages/opencode/src/altimate/review/post-github.ts b/packages/opencode/src/altimate/review/post-github.ts index bca5bb59c3..9bb445a773 100644 --- a/packages/opencode/src/altimate/review/post-github.ts +++ b/packages/opencode/src/altimate/review/post-github.ts @@ -64,6 +64,12 @@ export interface PostResult { } /** Headline of a review this bot posted (see verdictHeadline). */ +/** True when no engine call and no file failed, so missing findings mean "fixed", not "not checked". */ +export function runCompleted(env: VerdictEnvelope): boolean { + if (env.engineErrors?.length) return false + return !(env.scope?.unverified ?? []).some((u) => u.reason.startsWith("review failed for this file")) +} + const BOT_REVIEW_RE = /(Approved|Reviewed with comments|Changes requested)\b.*\((trivial|lite|full) tier/ /** The Octokit surface the poster uses; injectable so the posting rules @@ -131,7 +137,9 @@ export async function postGitHubReview( // Gate lifecycle: when this run no longer blocks, an earlier blocking // altimate review would keep the PR blocked forever. Dismiss it, pointing at // the new verdict. (Requires `pull-requests: write`; a failure is non-fatal.) - if (event === "COMMENT" && staleBlocking.length) { + // Only a run that actually completed may clear a block: when an engine call + // or a file failed, the blocking finding may simply not have been produced. + if (event === "COMMENT" && staleBlocking.length && runCompleted(env)) { let dismissed = 0 for (const id of staleBlocking) { try { diff --git a/packages/opencode/test/altimate/review-failures-gate.test.ts b/packages/opencode/test/altimate/review-failures-gate.test.ts index 07f2c4e252..3fcc45fc79 100644 --- a/packages/opencode/test/altimate/review-failures-gate.test.ts +++ b/packages/opencode/test/altimate/review-failures-gate.test.ts @@ -203,6 +203,29 @@ describe("posting: dedup, dismissal of a stale block, no repeat reviews", () => expect(r.dismissedReviews).toBe(1) }) + test("a rerun where the engine or a file failed does not clear the block", async () => { + const blocking = [{ id: 11, body: "🛑 Changes requested — 1 critical (full tier)", state: "CHANGES_REQUESTED" }] + const crashed = buildEnvelope({ + findings: [], + tier: "full", + mode: "gate", + engineErrors: [{ method: "check", reason: "native core not loaded", count: 3 }], + }) + const a = fakeClient({ reviews: blocking }) + await postGitHubReview(crashed, target, a.client) + expect(a.calls.dismissReview).toHaveLength(0) + + const fileFailed = buildEnvelope({ + findings: [], + tier: "full", + mode: "gate", + scope: { unverified: [{ path: "models/m.sql", reason: "review failed for this file: boom" }] } as any, + }) + const b = fakeClient({ reviews: blocking }) + await postGitHubReview(fileFailed, target, b.client) + expect(b.calls.dismissReview).toHaveLength(0) + }) + test("a still-blocking rerun does not dismiss and posts REQUEST_CHANGES", async () => { const blocking = makeFinding({ severity: "critical", diff --git a/packages/opencode/test/altimate/review-scope-diff.test.ts b/packages/opencode/test/altimate/review-scope-diff.test.ts index f2503321fc..88051f363b 100644 --- a/packages/opencode/test/altimate/review-scope-diff.test.ts +++ b/packages/opencode/test/altimate/review-scope-diff.test.ts @@ -431,3 +431,36 @@ describe("reviewPullRequest end to end: monorepo, PR into a non-default branch", expect(env.findings.some((f) => f.file.includes("drift"))).toBe(false) }, 60_000) }) + +describe("a shallow CI checkout without the target branch", () => { + test("the target branch and enough history are fetched to find the merge-base", async () => { + const exec = (await import("node:util")).promisify((await import("node:child_process")).execFile) + const fsp = (await import("node:fs")).promises + const os = await import("node:os") + const pth = await import("node:path") + const g = (cwd: string, ...a: string[]) => exec("git", a, { cwd }).then((r) => r.stdout.trim()) + const origin = await fsp.realpath(await fsp.mkdtemp(pth.join(os.tmpdir(), "shallow-origin-"))) + await g(origin, "init", "-q", "-b", "main") + await g(origin, "config", "user.email", "t@t") + await g(origin, "config", "user.name", "t") + await g(origin, "config", "commit.gpgsign", "false") + await g(origin, "config", "uploadpack.allowReachableSHA1InWant", "true") + for (let i = 0; i < 5; i++) { + await fsp.writeFile(pth.join(origin, `f${i}.sql`), `select ${i}\n`) + await g(origin, "add", "-A") + await g(origin, "commit", "-qm", `main ${i}`) + } + const branchPoint = await g(origin, "rev-parse", "HEAD") + await g(origin, "checkout", "-q", "-b", "feature") + for (let i = 0; i < 3; i++) { + await fsp.writeFile(pth.join(origin, `pr${i}.sql`), `select ${i}\n`) + await g(origin, "add", "-A") + await g(origin, "commit", "-qm", `pr ${i}`) + } + const clone = await fsp.realpath(await fsp.mkdtemp(pth.join(os.tmpdir(), "shallow-clone-"))) + await exec("git", ["clone", "-q", "--depth", "1", "--branch", "feature", `file://${origin}`, clone]) + const resolved = await resolveReviewBase({ cwd: clone, env: { GITHUB_BASE_REF: "main" } }) + expect(resolved.source).toBe("github_env") + expect(resolved.sha).toBe(branchPoint) + }, 60_000) +}) From 6fb241169600655f255adc4969ee23bc9b2ff426 Mon Sep 17 00:00:00 2001 From: RaghwendraSingh Date: Fri, 20 Nov 2026 10:21:08 +0530 Subject: [PATCH 3/4] fix(review): address review comments: engine `success:false`, per-file isolation, failed-file verdict, base PII abstention, bot-review identity - `runner.ts`: `altimate_core.*` handlers return `{ success: false }` instead of throwing; a `callCore` wrapper rethrows so each method records the failure in `engineErrors` and a failed check no longer reads as clean and PII-free - `orchestrate.ts`: the synchronous `lineageBreakageLane` / `detectModelPatterns` calls run as per-file tasks, and each schema.yml file is guarded, so one throw degrades only that file instead of aborting the review - `orchestrate.ts`: an unanalysable base no longer reports every head PII column as newly exposed; the file is marked unverified instead - `verdict.ts`: a file whose review failed downgrades an otherwise clean verdict to COMMENT (shared `FAILED_FILE_REASON`) - `post-github.ts`: stale blocking reviews must carry a hidden marker or a bot author, so a human review quoting the headline is never dismissed - `format.ts`: group unverified paths without re-spreading the array per item Co-Authored-By: Claude Opus 5.5 --- .../opencode/src/altimate/review/format.ts | 6 +- .../src/altimate/review/orchestrate.ts | 48 ++++++++---- .../src/altimate/review/post-github.ts | 23 ++++-- .../opencode/src/altimate/review/runner.ts | 32 +++++--- .../opencode/src/altimate/review/verdict.ts | 12 ++- .../altimate/review-failures-gate.test.ts | 73 +++++++++++++++++-- .../test/altimate/review-runner.test.ts | 19 +++++ 7 files changed, 172 insertions(+), 41 deletions(-) diff --git a/packages/opencode/src/altimate/review/format.ts b/packages/opencode/src/altimate/review/format.ts index c033c81fb0..48522a4a25 100644 --- a/packages/opencode/src/altimate/review/format.ts +++ b/packages/opencode/src/altimate/review/format.ts @@ -209,7 +209,11 @@ export function coverageNotes(env: VerdictEnvelope): string[] { if (undecided) notes.push(`${undecided} finding(s) could not be decided by the engine and are marked _unverified_.`) const unverified = env.scope?.unverified ?? [] const byReason = new Map() - for (const u of unverified) byReason.set(u.reason, [...(byReason.get(u.reason) ?? []), u.path]) + for (const u of unverified) { + const paths = byReason.get(u.reason) + if (paths) paths.push(u.path) + else byReason.set(u.reason, [u.path]) + } for (const [reason, paths] of byReason) { const head = paths.slice(0, 5).map((p) => `\`${p}\``).join(", ") const more = paths.length > 5 ? ` and ${paths.length - 5} more` : "" diff --git a/packages/opencode/src/altimate/review/orchestrate.ts b/packages/opencode/src/altimate/review/orchestrate.ts index b3df1e2571..a77022d837 100644 --- a/packages/opencode/src/altimate/review/orchestrate.ts +++ b/packages/opencode/src/altimate/review/orchestrate.ts @@ -12,7 +12,14 @@ import { type ChangedFile, type DbtProjectLayout, filterChangedFiles, hasOldSide import { classifyPR, compilePathTokenResolver, TIER_LANES } from "./risk-tier" import { type Rubric, exclusionReason, clampSeverity } from "./rubric" import { type ReviewConfig } from "./config" -import { type ReviewMode, type ReviewScope, type VerdictEnvelope, buildEnvelope, signEnvelope } from "./verdict" +import { + type ReviewMode, + type ReviewScope, + type VerdictEnvelope, + FAILED_FILE_REASON, + buildEnvelope, + signEnvelope, +} from "./verdict" import { detectModelPatterns, detectSchemaYmlPatterns, splitDiff } from "./dbt-patterns" import { type AiLaneStatus, type AiReviewInput, type AiReviewResult, safeReason } from "./ai-review" @@ -1171,7 +1178,7 @@ export async function runReview(input: OrchestrateInput): Promise unverified.push({ path: file.path, - reason: `review failed for this file: ${(err instanceof Error ? err.message : String(err)).replace(/\s+/g, " ").slice(0, 160)}`, + reason: `${FAILED_FILE_REASON}: ${(err instanceof Error ? err.message : String(err)).replace(/\s+/g, " ").slice(0, 160)}`, }) await Promise.all( modelFiles.map(async (file) => { @@ -1303,7 +1310,8 @@ export async function runReview(input: OrchestrateInput): Promise lineageBreakageLane(ctx.file, ctx.impact, input.rubric))) tasks.push(columnBreakageLane(ctx, input.runner, getCompiled, dialect)) } if (lanes.has("test_coverage")) tasks.push(missingGrainTestLane(ctx, input.runner)) @@ -1334,10 +1342,16 @@ export async function runReview(input: OrchestrateInput): Promise c.toLowerCase()), - ) - fallbackPii = fallbackPii.filter((col) => !basePii.has(col.toLowerCase())) + const baseRes = await input.runner.detectPii(currentCtx.engineOldSql, dialect) + if (baseRes.abstained) { + // An unanalysable base says nothing about what it exposed; reporting + // every head column as newly exposed would be a false positive. + unverified.push({ path: currentCtx.file.path, reason: "the PII check could not analyse the base model" }) + fallbackPii = [] + } else { + const basePii = new Set(baseRes.columns.map((c) => c.toLowerCase())) + fallbackPii = fallbackPii.filter((col) => !basePii.has(col.toLowerCase())) + } } const baseUnknown = hasOldSide(currentCtx.file) && !currentCtx.engineOldSql findings.push(...piiLane(currentCtx.file, fallbackPii, baseUnknown)) @@ -1347,7 +1361,7 @@ export async function runReview(input: OrchestrateInput): Promise detectModelPatterns(ctx.file, ctx.newSql, input.rubric))) tasks.push(lexicalLane(ctx.file, input.runner)) } // dbt config/Jinja lint (materialization / incremental / contract) — gate on its @@ -1415,12 +1429,18 @@ export async function runReview(input: OrchestrateInput): Promise { - const oldRef = file.oldPath ?? file.path - const [oldContent, newContent] = await Promise.all([ - file.status !== "added" ? getContent?.(oldRef, "old") : Promise.resolve(undefined), - file.status !== "deleted" ? getContent?.(file.path, "new") : Promise.resolve(undefined), - ]) - return detectSchemaYmlPatterns(file, input.rubric, { oldContent, newContent }) + try { + const oldRef = file.oldPath ?? file.path + const [oldContent, newContent] = await Promise.all([ + file.status !== "added" ? getContent?.(oldRef, "old") : Promise.resolve(undefined), + file.status !== "deleted" ? getContent?.(file.path, "new") : Promise.resolve(undefined), + ]) + return detectSchemaYmlPatterns(file, input.rubric, { oldContent, newContent }) + } catch (err) { + // One malformed schema file degrades that file only. + failedFile(file, err) + return [] as Finding[] + } }), ) for (const findings of schemaFindingSets) all.push(findings) diff --git a/packages/opencode/src/altimate/review/post-github.ts b/packages/opencode/src/altimate/review/post-github.ts index 9bb445a773..cea57c9326 100644 --- a/packages/opencode/src/altimate/review/post-github.ts +++ b/packages/opencode/src/altimate/review/post-github.ts @@ -1,6 +1,6 @@ import { Octokit } from "@octokit/rest" import { promises as fs } from "node:fs" -import { type VerdictEnvelope, VCS_EVENT } from "./verdict" +import { type VerdictEnvelope, VCS_EVENT, FAILED_FILE_REASON } from "./verdict" import { renderSummary, inlineComments, postedFindingIds, REVIEW_MARKER, verdictHeadline } from "./format" /** @@ -63,14 +63,24 @@ export interface PostResult { dismissedReviews?: number } -/** Headline of a review this bot posted (see verdictHeadline). */ /** True when no engine call and no file failed, so missing findings mean "fixed", not "not checked". */ export function runCompleted(env: VerdictEnvelope): boolean { if (env.engineErrors?.length) return false - return !(env.scope?.unverified ?? []).some((u) => u.reason.startsWith("review failed for this file")) + return !(env.scope?.unverified ?? []).some((u) => u.reason.startsWith(FAILED_FILE_REASON)) } +/** Headline of a review this bot posted (see verdictHeadline). */ const BOT_REVIEW_RE = /(Approved|Reviewed with comments|Changes requested)\b.*\((trivial|lite|full) tier/ +/** Hidden marker in the body of every review this bot posts. */ +const BOT_REVIEW_MARKER = "" + +/** A review this bot posted: the headline, plus the marker or a bot author + * (reviews posted before the marker existed). A human review that quotes the + * headline is never treated as the bot's. */ +function isBotReview(r: { body?: string | null; user?: { type?: string } | null }): boolean { + const body = r.body ?? "" + return BOT_REVIEW_RE.test(body) && (body.includes(BOT_REVIEW_MARKER) || r.user?.type === "Bot") +} /** The Octokit surface the poster uses; injectable so the posting rules * (dedup, dismissal, skip) are testable without the network. */ @@ -111,7 +121,8 @@ export async function postGitHubReview( // this map only translates verdict → GitHub event. APPROVE always maps to a // COMMENT event — the bot must never submit a formal GitHub approval. const event = VCS_EVENT[env.verdict] - const reviewBody = verdictHeadline(env) + const headline = verdictHeadline(env) + const reviewBody = `${headline}\n\n${BOT_REVIEW_MARKER}` // Rerun dedup: a finding already commented inline (same fingerprint, which // ignores line numbers) is not posted again on the next push. @@ -124,7 +135,7 @@ export async function postGitHubReview( octo.paginate(octo.rest.pulls.listReviews, { owner, repo, pull_number: prNumber, per_page: 100 }), ]) posted = postedFindingIds(reviewComments.map((c) => c.body)) - const botReviews = reviews.filter((r) => BOT_REVIEW_RE.test(r.body ?? "")) + const botReviews = reviews.filter(isBotReview) priorBotReview = botReviews.length > 0 staleBlocking = botReviews.filter((r) => r.state === "CHANGES_REQUESTED").map((r) => r.id) } catch { @@ -148,7 +159,7 @@ export async function postGitHubReview( repo, pull_number: prNumber, review_id: id, - message: `Superseded by a newer altimate review: ${reviewBody}`, + message: `Superseded by a newer altimate review: ${headline}`, }) dismissed++ } catch { diff --git a/packages/opencode/src/altimate/review/runner.ts b/packages/opencode/src/altimate/review/runner.ts index aa59b398ca..39999dd96b 100644 --- a/packages/opencode/src/altimate/review/runner.ts +++ b/packages/opencode/src/altimate/review/runner.ts @@ -112,6 +112,14 @@ export function createDispatcherRunner(opts: DispatcherRunnerOptions): ReviewRun const prev = failures.get(method) failures.set(method, { reason: prev?.reason ?? reason, count: (prev?.count ?? 0) + 1 }) } + // The altimate_core handlers catch their own exceptions and return + // `{ success: false, error }` instead of throwing. Rethrow so each method's + // catch records the failure; otherwise a failed call reads as a clean result. + const callCore: typeof Dispatcher.call = async (method, params, context) => { + const res: any = await Dispatcher.call(method, params, context) + if (res?.success === false) throw new Error(res.error || `${method} failed`) + return res + } let manifestPromise: Promise | undefined async function loadManifest(): Promise { @@ -207,7 +215,7 @@ export function createDispatcherRunner(opts: DispatcherRunnerOptions): ReviewRun : hasTables ? { ...schema, dialect } : { tables: { _altimate_lint_: { columns: [{ name: "_", type: "string" }] } }, version: "1", dialect } - const res = await Dispatcher.call("altimate_core.check", { sql, schema_context: schemaContext, base_sql: baseSql }) + const res = await callCore("altimate_core.check", { sql, schema_context: schemaContext, base_sql: baseSql }) const data = (res.data ?? {}) as Record // `altimate_core.check` is a composite: { validation, lint, safety }. // The AST anti-pattern findings live at data.lint.findings (each a @@ -323,7 +331,7 @@ export function createDispatcherRunner(opts: DispatcherRunnerOptions): ReviewRun async grade(sql: string): Promise { try { - const res = await Dispatcher.call("altimate_core.grade", { sql, schema_context: await resolveSchema() }) + const res = await callCore("altimate_core.grade", { sql, schema_context: await resolveSchema() }) const data = (res.data ?? {}) as Record return { grade: data.overall_grade ?? data.grade } } catch (err) { @@ -338,7 +346,7 @@ export function createDispatcherRunner(opts: DispatcherRunnerOptions): ReviewRun async grain(sql: string): Promise<{ group_by: string[]; dedup_partition: string[] }> { try { - const res = await Dispatcher.call("altimate_core.grain", { sql }) + const res = await callCore("altimate_core.grain", { sql }) const d = (res.data ?? {}) as Record return { group_by: asArray(d.group_by), dedup_partition: asArray(d.dedup_partition) } } catch (err) { @@ -353,7 +361,7 @@ export function createDispatcherRunner(opts: DispatcherRunnerOptions): ReviewRun ): Promise> { const out: any[] = [] try { - const res = await Dispatcher.call("altimate_core.dbt_config_lint", { sql: rawSql }) + const res = await callCore("altimate_core.dbt_config_lint", { sql: rawSql }) out.push(...asArray((res.data as any)?.findings)) } catch (err) { fail("dbtConfigLint", err) @@ -361,7 +369,7 @@ export function createDispatcherRunner(opts: DispatcherRunnerOptions): ReviewRun } if (oldRawSql) { try { - const res = await Dispatcher.call("altimate_core.dbt_config_diff", { base_sql: oldRawSql, head_sql: rawSql }) + const res = await callCore("altimate_core.dbt_config_diff", { base_sql: oldRawSql, head_sql: rawSql }) out.push(...asArray((res.data as any)?.findings)) } catch (err) { fail("dbtConfigLint", err) @@ -376,7 +384,7 @@ export function createDispatcherRunner(opts: DispatcherRunnerOptions): ReviewRun headSql: string, ): Promise> { try { - const res = await Dispatcher.call("altimate_core.structural_diff", { base_sql: baseSql, head_sql: headSql }) + const res = await callCore("altimate_core.structural_diff", { base_sql: baseSql, head_sql: headSql }) return asArray((res.data as any)?.findings) } catch (err) { fail("structuralDiff", err) @@ -386,7 +394,7 @@ export function createDispatcherRunner(opts: DispatcherRunnerOptions): ReviewRun async sourceFilters(sql: string): Promise> { try { - const res = await Dispatcher.call("altimate_core.source_filters", { sql }) + const res = await callCore("altimate_core.source_filters", { sql }) const d = (res.data ?? {}) as Record return (d.filters ?? {}) as Record } catch (err) { @@ -416,7 +424,7 @@ export function createDispatcherRunner(opts: DispatcherRunnerOptions): ReviewRun async isComplex(sql: string): Promise { try { - const res = await Dispatcher.call("altimate_core.metadata", { sql }) + const res = await callCore("altimate_core.metadata", { sql }) const d = (res.data ?? {}) as Record return !!(d.has_window_functions || d.has_subqueries || (typeof d.node_count === "number" && d.node_count >= 12)) } catch (err) { @@ -430,7 +438,7 @@ export function createDispatcherRunner(opts: DispatcherRunnerOptions): ReviewRun try { // classify_pii classifies a SCHEMA's columns by name/type — feed it the // model's output columns as a one-table schema. - const res = await Dispatcher.call("altimate_core.classify_pii", { + const res = await callCore("altimate_core.classify_pii", { schema_context: { tables: { model: { columns: columns.map((name) => ({ name, type: "string" })) } }, version: "1", @@ -482,7 +490,7 @@ export function createDispatcherRunner(opts: DispatcherRunnerOptions): ReviewRun async columnLineage(sql: string, dialect?: string): Promise> { try { - const res = await Dispatcher.call("altimate_core.column_lineage", { + const res = await callCore("altimate_core.column_lineage", { sql, schema_path: "", schema_context: await resolveSchema(), @@ -502,7 +510,7 @@ export function createDispatcherRunner(opts: DispatcherRunnerOptions): ReviewRun async lexicalScan(addedLines: string[]) { if (!addedLines.length) return [] try { - const res = await Dispatcher.call("altimate_core.review_lexical_scan", { added_lines: addedLines }) + const res = await callCore("altimate_core.review_lexical_scan", { added_lines: addedLines }) const data = (res.data ?? {}) as Record return asArray(data.findings).map((f) => ({ rule: String(f.rule ?? "lexical"), @@ -519,7 +527,7 @@ export function createDispatcherRunner(opts: DispatcherRunnerOptions): ReviewRun async equivalence(oldSql: string, newSql: string, dialect?: string): Promise { try { const schema = await resolveSchema() - const res = await Dispatcher.call("altimate_core.equivalence", { + const res = await callCore("altimate_core.equivalence", { sql1: oldSql, sql2: newSql, dialect, diff --git a/packages/opencode/src/altimate/review/verdict.ts b/packages/opencode/src/altimate/review/verdict.ts index b30d5f5fc4..bdc06288b0 100644 --- a/packages/opencode/src/altimate/review/verdict.ts +++ b/packages/opencode/src/altimate/review/verdict.ts @@ -249,13 +249,19 @@ function summarize(findings: Finding[], degraded: boolean): VerdictEnvelope["sum return { critical: tally.critical, warning: tally.warning, suggestion: tally.suggestion, degraded } } +/** Prefix of the `scope.unverified` reason for a file whose review threw. */ +export const FAILED_FILE_REASON = "review failed for this file" + /** Assemble the verdict envelope (unsigned). Call signEnvelope to sign it. */ export function buildEnvelope(input: BuildEnvelopeInput): VerdictEnvelope { const rubric = input.rubric ?? DEFAULT_RUBRIC - // An engine failure means some checks did not run: "no findings" is then - // unknown, not clean, so the verdict cannot be APPROVE. + // An engine failure or a file whose review failed means some checks did not + // run: "no findings" is then unknown, not clean, so the verdict cannot be APPROVE. const computed = computeIdealVerdict(input.findings, rubric) - const ideal = computed === "APPROVE" && input.engineErrors?.length ? "COMMENT" : computed + const incomplete = + !!input.engineErrors?.length || + (input.scope?.unverified ?? []).some((u) => u.reason.startsWith(FAILED_FILE_REASON)) + const ideal = computed === "APPROVE" && incomplete ? "COMMENT" : computed const verdict = applyMode(ideal, input.mode) const degraded = input.degraded ?? input.findings.some((f) => f.degraded) return VerdictEnvelope.parse({ diff --git a/packages/opencode/test/altimate/review-failures-gate.test.ts b/packages/opencode/test/altimate/review-failures-gate.test.ts index 3fcc45fc79..38ee065e37 100644 --- a/packages/opencode/test/altimate/review-failures-gate.test.ts +++ b/packages/opencode/test/altimate/review-failures-gate.test.ts @@ -113,7 +113,51 @@ describe("one file's failure degrades only that file", () => { }) }) +describe("a file whose review failed is never approved", () => { + test("a failed-file entry downgrades an otherwise clean verdict to COMMENT", () => { + const env = buildEnvelope({ + findings: [], + tier: "lite", + mode: "comment", + scope: { unverified: [{ path: "models/a.sql", reason: "review failed for this file: boom" }] } as any, + }) + expect(env.idealVerdict).toBe("COMMENT") + }) + + test("a synchronous detector that throws degrades only its file", async () => { + const env = await runReview({ + changedFiles: [{ path: "models/schema.yml", kind: "schema_yml", status: "modified" as const, diff: "- - unique\n" }] as any, + config: { ...DEFAULT_REVIEW_CONFIG, reviewers: ["sql_quality"] }, + rubric: DEFAULT_RUBRIC, + mode: "comment", + runner: runner(), + getContent: async () => { + throw new Error("git show failed") + }, + }) + expect(env.scope?.unverified).toEqual([{ path: "models/schema.yml", reason: "review failed for this file: git show failed" }]) + expect(env.idealVerdict).toBe("COMMENT") + }) +}) + describe("PII abstention", () => { + test("an unanalysable base does not report every head PII column as newly exposed", async () => { + const env = await runReview({ + changedFiles: [{ path: "models/a.sql", status: "modified" as const, diff: "-select 1 as id\n+select email from users\n" }], + config: { ...DEFAULT_REVIEW_CONFIG, reviewers: ["pii_exposure"] }, + rubric: DEFAULT_RUBRIC, + mode: "comment", + runner: runner({ + async detectPii(sql: string) { + return sql.includes("email") ? { columns: ["email"] } : { columns: [], abstained: true } + }, + }), + getContent: async (_f, side) => (side === "new" ? "select email from users" : "select 1 as id"), + }) + expect(env.findings.filter((f) => f.category === "pii_exposure")).toHaveLength(0) + expect(env.scope?.unverified).toContainEqual({ path: "models/a.sql", reason: "the PII check could not analyse the base model" }) + }) + test("a PII check that could not analyse the model is reported, not treated as clean", async () => { const env = await runReview({ changedFiles: [two[0]], @@ -135,7 +179,7 @@ describe("PII abstention", () => { describe("posting: dedup, dismissal of a stale block, no repeat reviews", () => { function fakeClient(state: { reviewComments?: Array<{ body: string }> - reviews?: Array<{ id: number; body: string; state: string }> + reviews?: Array<{ id: number; body: string; state: string; user?: { type: string } }> }) { const calls: Record = { createReview: [], dismissReview: [], createComment: [], updateComment: [] } const client = { @@ -177,6 +221,7 @@ describe("posting: dedup, dismissal of a stale block, no repeat reviews", () => return { client: client as unknown as GitHubClient, calls } } const target = { token: "t", owner: "o", repo: "r", prNumber: 7 } + const bot = { type: "Bot" } const engineFinding = (line?: number) => makeFinding({ severity: "warning", @@ -194,7 +239,7 @@ describe("posting: dedup, dismissal of a stale block, no repeat reviews", () => const env = buildEnvelope({ findings: [], tier: "lite", mode: "gate" }) const { client, calls } = fakeClient({ reviews: [ - { id: 11, body: "🛑 Changes requested — 1 critical (full tier)", state: "CHANGES_REQUESTED" }, + { id: 11, body: "🛑 Changes requested — 1 critical (full tier)", state: "CHANGES_REQUESTED", user: bot }, { id: 12, body: "LGTM from a human", state: "CHANGES_REQUESTED" }, ], }) @@ -203,8 +248,25 @@ describe("posting: dedup, dismissal of a stale block, no repeat reviews", () => expect(r.dismissedReviews).toBe(1) }) + test("a human review quoting the headline is never dismissed; the marker identifies the bot", async () => { + const env = buildEnvelope({ findings: [], tier: "lite", mode: "gate" }) + const { client, calls } = fakeClient({ + reviews: [ + { id: 21, body: "Changes requested (full tier) until the backfill is fixed", state: "CHANGES_REQUESTED", user: { type: "User" } }, + { + id: 22, + body: "🛑 Changes requested — 1 critical (full tier)\n\n", + state: "CHANGES_REQUESTED", + user: { type: "User" }, + }, + ], + }) + await postGitHubReview(env, target, client) + expect(calls.dismissReview.map((c) => c.review_id)).toEqual([22]) + }) + test("a rerun where the engine or a file failed does not clear the block", async () => { - const blocking = [{ id: 11, body: "🛑 Changes requested — 1 critical (full tier)", state: "CHANGES_REQUESTED" }] + const blocking = [{ id: 11, body: "🛑 Changes requested — 1 critical (full tier)", state: "CHANGES_REQUESTED", user: bot }] const crashed = buildEnvelope({ findings: [], tier: "full", @@ -237,11 +299,12 @@ describe("posting: dedup, dismissal of a stale block, no repeat reviews", () => }) const env = buildEnvelope({ findings: [blocking], tier: "full", mode: "gate" }) const { client, calls } = fakeClient({ - reviews: [{ id: 11, body: "🛑 Changes requested — 1 critical (full tier)", state: "CHANGES_REQUESTED" }], + reviews: [{ id: 11, body: "🛑 Changes requested — 1 critical (full tier)", state: "CHANGES_REQUESTED", user: bot }], }) await postGitHubReview(env, target, client) expect(calls.dismissReview).toHaveLength(0) expect(calls.createReview[0].event).toBe("REQUEST_CHANGES") + expect(calls.createReview[0].body).toContain("") }) test("findings already on the PR are not re-posted, and with nothing new no review is added", async () => { @@ -249,7 +312,7 @@ describe("posting: dedup, dismissal of a stale block, no repeat reviews", () => const env = buildEnvelope({ findings: [f], tier: "lite", mode: "comment" }) const { client, calls } = fakeClient({ reviewComments: [{ body: `⚠️ old\n\n${findingMarker(f.id)}` }], - reviews: [{ id: 5, body: "💬 Reviewed with comments — 1 warning (lite tier)", state: "COMMENTED" }], + reviews: [{ id: 5, body: "💬 Reviewed with comments — 1 warning (lite tier)", state: "COMMENTED", user: bot }], }) const r = await postGitHubReview(env, target, client) expect(r.inlineSkippedAsPosted).toBe(1) diff --git a/packages/opencode/test/altimate/review-runner.test.ts b/packages/opencode/test/altimate/review-runner.test.ts index 538b2d7631..de07bc6ddc 100644 --- a/packages/opencode/test/altimate/review-runner.test.ts +++ b/packages/opencode/test/altimate/review-runner.test.ts @@ -162,3 +162,22 @@ describe("runner honors engine `decidable` flag (core 0.5.1)", () => { await tmp[Symbol.asyncDispose]?.() }) }) + +describe("engine handlers that report failure instead of throwing", () => { + test("a {success:false} result is an engine failure, not a clean or PII-free check", async () => { + await using tmp = await tmpdir() + dispatcherSpy = spyOn(Dispatcher, "call").mockImplementation((async () => ({ + success: false, + data: {}, + error: "Error: native core not loaded", + })) as any) + const runner = createDispatcherRunner({ manifestPath: path.join(tmp.path, "missing.json") }) + expect((await runner.check("select 1 as id", "duckdb")).ran).toBe(false) + expect((await runner.detectPii("select 1 as id", "duckdb")).abstained).toBe(true) + expect(await runner.grain!("select 1 as id")).toEqual({ group_by: [], dedup_partition: [] }) + const failed = runner.failures!().map((f) => f.method) + expect(failed).toContain("check") + expect(failed).toContain("grain") + expect(runner.failures!()[0].reason).toBe("Error: native core not loaded") + }) +}) From 27a9a996d104202fdf551096ac89a183a7ddda81 Mon Sep 17 00:00:00 2001 From: RaghwendraSingh Date: Fri, 20 Nov 2026 11:13:29 +0530 Subject: [PATCH 4/4] fix(review): address review feedback: PII abstention keeps the block, typed unverified kinds, safer dismissal, deepen `--head` - `scope.unverified` entries carry a typed `kind` (`failed`, `abstained`, `not_compiled`). `reviewIncomplete` (engine errors, failed files, abstained checks) drives both the APPROVE -> COMMENT downgrade and `runCompleted`, so a PII abstention neither approves nor dismisses an earlier block. - Dismissal: only when this run's head (`scope.headSha`, new) is the PR's current head; only reviews carrying this project's marker (`` below the repo root); and only after the replacement review was posted (or skipped). A failed `createReview` keeps the block. - `fetchTargetHistory` deepens the configured head ref, not `HEAD`, and a shallow checkout starts with `--depth=50` instead of the target's whole history. - Runner: `dataDiff` records an in-band `success: false`; the config-diff failure is named `dbtConfigDiff`; stale `/* skip */` comments removed. - One `errorReason` helper (one line, bounded) for runner failures, lane errors and failed files, so a multi-line message cannot break the note. - Docs: abstentions block APPROVE and keep the block; dismissal rules. - Tests: verdict on unreadable content and PII abstention, a block surviving abstention, older-push and other-project runs not dismissing, no dismissal when the replacement review fails, deepening past 50 commits with `--head`, `dataDiff` in-band failure. Co-Authored-By: Claude Opus 5.5 --- docs/docs/usage/dbt-pr-review.md | 18 ++-- packages/opencode/src/altimate/review/git.ts | 30 +++++- .../src/altimate/review/orchestrate.ts | 25 +++-- .../src/altimate/review/post-github.ts | 69 +++++++++----- packages/opencode/src/altimate/review/run.ts | 3 + .../opencode/src/altimate/review/runner.ts | 13 ++- .../opencode/src/altimate/review/verdict.ts | 35 +++++-- .../altimate/review-failures-gate.test.ts | 91 ++++++++++++++++++- .../test/altimate/review-runner.test.ts | 2 + .../test/altimate/review-scope-diff.test.ts | 31 +++++++ 10 files changed, 256 insertions(+), 61 deletions(-) diff --git a/docs/docs/usage/dbt-pr-review.md b/docs/docs/usage/dbt-pr-review.md index de60bbb378..077bf0b55e 100644 --- a/docs/docs/usage/dbt-pr-review.md +++ b/docs/docs/usage/dbt-pr-review.md @@ -308,7 +308,8 @@ rubric: PII check. Now pattern-rule findings comment but don't block. To keep a pattern rule blocking, add its id to `rubric.blockingRules`. A blocking review is dismissed only by a later run that completed: when an engine call - or a file failed, the earlier block stays. + or a file failed, or a check (such as PII) could not analyse a model, the + earlier block stays. ### What the PR comment shows @@ -328,12 +329,15 @@ rubric: SQL; when it can't be compiled, PII columns are reported as non-blocking warnings that may predate the PR. A model the PII check could not analyse is listed as not fully verified. -- **Failures are never a clean pass:** if an engine check throws, the comment - names it and the verdict is at most COMMENT, never APPROVE (envelope - `engineErrors`). A failure on one file marks that file unverified and the rest - of the PR is still reviewed. -- **The gate clears itself:** when a rerun no longer blocks, the earlier blocking - altimate review is dismissed (needs `pull-requests: write`). +- **Failures are never a clean pass:** if an engine check fails, a file's + review fails, or a check could not analyse a model, the comment names it and + the verdict is at most COMMENT, never APPROVE (envelope `engineErrors` and + `scope.unverified`). A failure on one file marks that file unverified and the + rest of the PR is still reviewed. +- **The gate clears itself:** when a completed rerun on the PR's current head + no longer blocks, the earlier blocking altimate review for the same dbt + project is dismissed after the new review is posted (needs + `pull-requests: write`). ### Risk-tier path tokens diff --git a/packages/opencode/src/altimate/review/git.ts b/packages/opencode/src/altimate/review/git.ts index 055842f958..a21152f9d2 100644 --- a/packages/opencode/src/altimate/review/git.ts +++ b/packages/opencode/src/altimate/review/git.ts @@ -208,18 +208,29 @@ async function mergeBase(a: string, b: string, cwd: string): Promise Promise): Promise { +async function fetchTargetHistory( + branch: string, + head: string, + cwd: string, + found: () => Promise, +): Promise { if (branch.startsWith("-")) return false + const shallow = async () => (await git(["rev-parse", "--is-shallow-repository"], cwd)).trim() === "true" try { - await git(["fetch", "--no-tags", "origin", `+refs/heads/${branch}:refs/remotes/origin/${branch}`], cwd) + // In a shallow checkout, fetch only recent target history; the loop below + // deepens it. A full fetch would pull the target branch's whole history. + const depth = (await shallow()) ? ["--depth=50"] : [] + await git(["fetch", "--no-tags", ...depth, "origin", `+refs/heads/${branch}:refs/remotes/origin/${branch}`], cwd) } catch { return false } if (await found()) return true for (const depth of [50, 200, 1000]) { try { - if ((await git(["rev-parse", "--is-shallow-repository"], cwd)).trim() !== "true") return false - const headSha = (await git(["rev-parse", "HEAD"], cwd)).trim() + if (!(await shallow())) return false + // Deepen the history of the head being reviewed, which may differ from + // the checkout (`--head `). + const headSha = (await git(["rev-parse", "--verify", `${head}^{commit}`], cwd)).trim() await git(["fetch", "--no-tags", `--deepen=${depth}`, "origin", `+refs/heads/${branch}:refs/remotes/origin/${branch}`, headSha], cwd) } catch { return false @@ -275,7 +286,7 @@ export async function resolveReviewBase(opts: ResolveBaseOptions): Promise { + try { + return (await git(["rev-parse", "--verify", "--quiet", `${ref}^{commit}`], cwd)).trim() || undefined + } catch { + return undefined + } +} + /** Compute a short hash of the manifest file for the verdict envelope. */ export async function manifestHash(manifestPath: string, cwd: string): Promise { try { diff --git a/packages/opencode/src/altimate/review/orchestrate.ts b/packages/opencode/src/altimate/review/orchestrate.ts index a77022d837..764fb45c98 100644 --- a/packages/opencode/src/altimate/review/orchestrate.ts +++ b/packages/opencode/src/altimate/review/orchestrate.ts @@ -18,6 +18,7 @@ import { type VerdictEnvelope, FAILED_FILE_REASON, buildEnvelope, + errorReason, signEnvelope, } from "./verdict" import { detectModelPatterns, detectSchemaYmlPatterns, splitDiff } from "./dbt-patterns" @@ -1163,7 +1164,7 @@ export async function runReview(input: OrchestrateInput): Promise f.kind === "model_sql" || f.kind === "python_model") const ctxByPath = new Map() - const unverified: Array<{ path: string; reason: string }> = [] + const unverified: NonNullable = [] let anyManifest = false if (input.runner.manifestAvailable) { anyManifest = await input.runner.manifestAvailable().catch(() => false) @@ -1172,13 +1173,14 @@ export async function runReview(input: OrchestrateInput): Promise = [] const prLevel = (method: string, lane: Promise) => lane.catch((err) => { - laneErrors.push({ method, reason: (err instanceof Error ? err.message : String(err)).slice(0, 200), count: 1 }) + laneErrors.push({ method, reason: errorReason(err, 200), count: 1 }) return [] as Finding[] }) const failedFile = (file: { path: string }, err: unknown) => unverified.push({ path: file.path, - reason: `${FAILED_FILE_REASON}: ${(err instanceof Error ? err.message : String(err)).replace(/\s+/g, " ").slice(0, 160)}`, + reason: `${FAILED_FILE_REASON}: ${errorReason(err, 160)}`, + kind: "failed", }) await Promise.all( modelFiles.map(async (file) => { @@ -1202,13 +1204,18 @@ export async function runReview(input: OrchestrateInput): Promise c.toLowerCase())) diff --git a/packages/opencode/src/altimate/review/post-github.ts b/packages/opencode/src/altimate/review/post-github.ts index cea57c9326..3b9699e325 100644 --- a/packages/opencode/src/altimate/review/post-github.ts +++ b/packages/opencode/src/altimate/review/post-github.ts @@ -1,6 +1,6 @@ import { Octokit } from "@octokit/rest" import { promises as fs } from "node:fs" -import { type VerdictEnvelope, VCS_EVENT, FAILED_FILE_REASON } from "./verdict" +import { type VerdictEnvelope, VCS_EVENT, reviewIncomplete } from "./verdict" import { renderSummary, inlineComments, postedFindingIds, REVIEW_MARKER, verdictHeadline } from "./format" /** @@ -63,23 +63,32 @@ export interface PostResult { dismissedReviews?: number } -/** True when no engine call and no file failed, so missing findings mean "fixed", not "not checked". */ +/** True when every check ran (no engine error, no failed file, no abstained + * check), so missing findings mean "fixed", not "not checked". */ export function runCompleted(env: VerdictEnvelope): boolean { - if (env.engineErrors?.length) return false - return !(env.scope?.unverified ?? []).some((u) => u.reason.startsWith(FAILED_FILE_REASON)) + return !reviewIncomplete(env) } /** Headline of a review this bot posted (see verdictHeadline). */ const BOT_REVIEW_RE = /(Approved|Reviewed with comments|Changes requested)\b.*\((trivial|lite|full) tier/ -/** Hidden marker in the body of every review this bot posts. */ -const BOT_REVIEW_MARKER = "" +/** Hidden marker in the body of every review this bot posts; reviews of a + * dbt project below the repository root also name it (`project=`). */ +const BOT_REVIEW_MARKER_RE = // -/** A review this bot posted: the headline, plus the marker or a bot author - * (reviews posted before the marker existed). A human review that quotes the - * headline is never treated as the bot's. */ -function isBotReview(r: { body?: string | null; user?: { type?: string } | null }): boolean { +function botReviewMarker(projectRoot: string | undefined): string { + return projectRoot ? `` : "" +} + +/** A review this bot posted for this dbt project: the headline, plus the + * marker or a bot author (reviews posted before the marker existed). A human + * review that quotes the headline is never treated as the bot's, and in a + * monorepo one project's run never touches another project's review. */ +function isBotReview(r: { body?: string | null; user?: { type?: string } | null }, projectRoot: string | undefined): boolean { const body = r.body ?? "" - return BOT_REVIEW_RE.test(body) && (body.includes(BOT_REVIEW_MARKER) || r.user?.type === "Bot") + if (!BOT_REVIEW_RE.test(body)) return false + const marker = BOT_REVIEW_MARKER_RE.exec(body) + if (!marker && r.user?.type !== "Bot") return false + return (marker?.[1] ?? "") === (projectRoot ?? "") } /** The Octokit surface the poster uses; injectable so the posting rules @@ -122,7 +131,8 @@ export async function postGitHubReview( // COMMENT event — the bot must never submit a formal GitHub approval. const event = VCS_EVENT[env.verdict] const headline = verdictHeadline(env) - const reviewBody = `${headline}\n\n${BOT_REVIEW_MARKER}` + const projectRoot = env.scope?.projectRoot || undefined + const reviewBody = `${headline}\n\n${botReviewMarker(projectRoot)}` // Rerun dedup: a finding already commented inline (same fingerprint, which // ignores line numbers) is not posted again on the next push. @@ -135,7 +145,7 @@ export async function postGitHubReview( octo.paginate(octo.rest.pulls.listReviews, { owner, repo, pull_number: prNumber, per_page: 100 }), ]) posted = postedFindingIds(reviewComments.map((c) => c.body)) - const botReviews = reviews.filter(isBotReview) + const botReviews = reviews.filter((r) => isBotReview(r, projectRoot)) priorBotReview = botReviews.length > 0 staleBlocking = botReviews.filter((r) => r.state === "CHANGES_REQUESTED").map((r) => r.id) } catch { @@ -145,12 +155,27 @@ export async function postGitHubReview( const withLine = env.findings.filter((f) => typeof f.startLine === "number").length result.inlineSkippedAsPosted = withLine - comments.length + // Anchor inline comments to the PR head commit. Without commit_id, GitHub's + // line-positioning is fragile and can 422 even on valid lines. + let commitId: string | undefined + try { + const pr = await octo.rest.pulls.get({ owner, repo, pull_number: prNumber }) + commitId = pr.data.head.sha + } catch { + /* fall back to default head resolution */ + } + // Gate lifecycle: when this run no longer blocks, an earlier blocking // altimate review would keep the PR blocked forever. Dismiss it, pointing at // the new verdict. (Requires `pull-requests: write`; a failure is non-fatal.) // Only a run that actually completed may clear a block: when an engine call - // or a file failed, the blocking finding may simply not have been produced. - if (event === "COMMENT" && staleBlocking.length && runCompleted(env)) { + // or a file failed, or a check abstained, the blocking finding may simply + // not have been produced. A run on an older push (its head is no longer the + // PR's head) never clears the block a newer push's run may have posted. + const headSha = env.scope?.headSha + const onCurrentHead = !headSha || !commitId || headSha === commitId + const dismissStale = async () => { + if (event !== "COMMENT" || !staleBlocking.length || !runCompleted(env) || !onCurrentHead) return let dismissed = 0 for (const id of staleBlocking) { try { @@ -174,19 +199,10 @@ export async function postGitHubReview( // comments" entry per push is noise. A blocking verdict always posts. if (!comments.length && event === "COMMENT" && priorBotReview) { result.reviewSkipped = true + await dismissStale() return result } - // Anchor inline comments to the PR head commit. Without commit_id, GitHub's - // line-positioning is fragile and can 422 even on valid lines. - let commitId: string | undefined - try { - const pr = await octo.rest.pulls.get({ owner, repo, pull_number: prNumber }) - commitId = pr.data.head.sha - } catch { - /* fall back to default head resolution */ - } - try { const r = await octo.rest.pulls.createReview({ owner, @@ -198,6 +214,7 @@ export async function postGitHubReview( comments: comments.length ? comments : undefined, }) result.reviewId = r.data.id + await dismissStale() } catch (e: any) { // Only a line-positioning error (422 Unprocessable) justifies dropping the // inline comments — anchoring outside the diff hunk. Other errors (auth, @@ -221,7 +238,9 @@ export async function postGitHubReview( body: reviewBody, }) result.reviewId = r.data.id + await dismissStale() } catch (e2: any) { + // The replacement review did not land: keep the earlier block. // Summary comment already landed; don't fail CI on a review-post error. result.postError = `${e2?.status ?? ""} ${e2?.message ?? e2}`.trim() } diff --git a/packages/opencode/src/altimate/review/run.ts b/packages/opencode/src/altimate/review/run.ts index 4f9eb7642f..556b15db7d 100644 --- a/packages/opencode/src/altimate/review/run.ts +++ b/packages/opencode/src/altimate/review/run.ts @@ -10,6 +10,7 @@ import { type ResolvedBase, gitRepoRoot, manifestHash, + resolveCommit, showAtRef, } from "./git" import { makeCompiledResolver, dbtProjectName, dbtProjectLayout } from "./compiled" @@ -237,6 +238,7 @@ export async function reviewPullRequest(opts: ReviewPullRequestOptions): Promise const resolved: ResolvedBase | undefined = needGit ? await resolveReviewBase({ cwd: opts.cwd, base: opts.base, head: opts.head }) : undefined + const headSha = resolved ? await resolveCommit(opts.head ?? "HEAD", opts.cwd) : undefined // The policy (.altimate/review.yml) comes from the BASE commit, so the PR // under review cannot relax its own rules; what it would change is listed. @@ -436,6 +438,7 @@ export async function reviewPullRequest(opts: ReviewPullRequestOptions): Promise scope: { baseRef: resolved?.ref ?? opts.base, baseSha: resolved?.sha, + headSha, baseSource: resolved?.source, projectRoot: projectRootRel, outOfScopeFiles: scoped.outOfScope || undefined, diff --git a/packages/opencode/src/altimate/review/runner.ts b/packages/opencode/src/altimate/review/runner.ts index 39999dd96b..cb4126f435 100644 --- a/packages/opencode/src/altimate/review/runner.ts +++ b/packages/opencode/src/altimate/review/runner.ts @@ -3,6 +3,7 @@ import { EngineCoerce } from "../native/engine-coerce" import { parseManifest } from "../native/dbt/manifest" import type { CheckResult, EquivalenceResult, GradeResult, ImpactResult, ReviewRunner } from "./orchestrate" import { buildReviewSchemaContext, type SchemaContext } from "./schema-context" +import { errorReason } from "./verdict" /** * Production ReviewRunner backed by the native Dispatcher (the Rust core). @@ -108,7 +109,7 @@ export function createDispatcherRunner(opts: DispatcherRunnerOptions): ReviewRun // instead of reading as "no findings". const failures = new Map() function fail(method: string, err: unknown) { - const reason = (err instanceof Error ? err.message : String(err)).replace(/\s+/g, " ").slice(0, 200) + const reason = errorReason(err) const prev = failures.get(method) failures.set(method, { reason: prev?.reason ?? reason, count: (prev?.count ?? 0) + 1 }) } @@ -365,15 +366,13 @@ export function createDispatcherRunner(opts: DispatcherRunnerOptions): ReviewRun out.push(...asArray((res.data as any)?.findings)) } catch (err) { fail("dbtConfigLint", err) - /* skip */ } if (oldRawSql) { try { const res = await callCore("altimate_core.dbt_config_diff", { base_sql: oldRawSql, head_sql: rawSql }) out.push(...asArray((res.data as any)?.findings)) } catch (err) { - fail("dbtConfigLint", err) - /* skip */ + fail("dbtConfigDiff", err) } } return out @@ -474,7 +473,11 @@ export function createDispatcherRunner(opts: DispatcherRunnerOptions): ReviewRun target_warehouse: warehouse || undefined, }) const r = res as any - if (!r || r.success === false) return null + if (r?.success === false) { + fail("dataDiff", r.error ?? "data.diff failed") + return null + } + if (!r) return null const o = (r.outcome ?? {}) as Record return { rowsOnlyInBase: o.rows_only_in_source ?? o.removed ?? o.exclusive_source, diff --git a/packages/opencode/src/altimate/review/verdict.ts b/packages/opencode/src/altimate/review/verdict.ts index bdc06288b0..6b93b14701 100644 --- a/packages/opencode/src/altimate/review/verdict.ts +++ b/packages/opencode/src/altimate/review/verdict.ts @@ -118,8 +118,14 @@ export const ReviewScope = z.object({ outOfScopeFiles: z.number().int().nonnegative().optional(), /** Non-model renames with unchanged content, name, role and layer (pure moves): not re-reviewed. */ movedUnchangedFiles: z.number().int().nonnegative().optional(), - /** Models the engine lanes could not verify, with the reason. */ - unverified: z.array(z.object({ path: z.string(), reason: z.string() })).optional(), + /** Head commit the review ran on; a run on an older push never clears a block. */ + headSha: z.string().optional(), + /** Models the engine lanes could not verify, with the reason and its kind: + * `failed` (the file's review threw), `abstained` (a check could not analyse + * the SQL), `not_compiled` (no compiled SQL; engine lanes skipped). */ + unverified: z + .array(z.object({ path: z.string(), reason: z.string(), kind: z.enum(["failed", "abstained", "not_compiled"]).optional() })) + .optional(), }) export type ReviewScope = z.infer @@ -249,19 +255,32 @@ function summarize(findings: Finding[], degraded: boolean): VerdictEnvelope["sum return { critical: tally.critical, warning: tally.warning, suggestion: tally.suggestion, degraded } } +/** An error as one bounded line, for envelope reasons and the PR comment. */ +export function errorReason(err: unknown, max = 200): string { + return (err instanceof Error ? err.message : String(err)).replace(/\s+/g, " ").trim().slice(0, max) +} + /** Prefix of the `scope.unverified` reason for a file whose review threw. */ export const FAILED_FILE_REASON = "review failed for this file" +/** True when some check did not run: an engine call failed, a file's review + * threw, or a check abstained on a model. Missing findings then mean "not + * checked", so the run can neither APPROVE nor clear an earlier block. */ +export function reviewIncomplete(input: Pick): boolean { + if (input.engineErrors?.length) return true + return (input.scope?.unverified ?? []).some( + (u) => u.kind === "failed" || u.kind === "abstained" || u.reason.startsWith(FAILED_FILE_REASON), + ) +} + /** Assemble the verdict envelope (unsigned). Call signEnvelope to sign it. */ export function buildEnvelope(input: BuildEnvelopeInput): VerdictEnvelope { const rubric = input.rubric ?? DEFAULT_RUBRIC - // An engine failure or a file whose review failed means some checks did not - // run: "no findings" is then unknown, not clean, so the verdict cannot be APPROVE. + // An engine failure, a file whose review failed, or a check that abstained + // means some checks did not run: "no findings" is then unknown, not clean, + // so the verdict cannot be APPROVE. const computed = computeIdealVerdict(input.findings, rubric) - const incomplete = - !!input.engineErrors?.length || - (input.scope?.unverified ?? []).some((u) => u.reason.startsWith(FAILED_FILE_REASON)) - const ideal = computed === "APPROVE" && incomplete ? "COMMENT" : computed + const ideal = computed === "APPROVE" && reviewIncomplete(input) ? "COMMENT" : computed const verdict = applyMode(ideal, input.mode) const degraded = input.degraded ?? input.findings.some((f) => f.degraded) return VerdictEnvelope.parse({ diff --git a/packages/opencode/test/altimate/review-failures-gate.test.ts b/packages/opencode/test/altimate/review-failures-gate.test.ts index 38ee065e37..dac4afa702 100644 --- a/packages/opencode/test/altimate/review-failures-gate.test.ts +++ b/packages/opencode/test/altimate/review-failures-gate.test.ts @@ -90,8 +90,11 @@ describe("one file's failure degrades only that file", () => { }) expect(checked).toContain("select 2 as id") expect(env.scope?.unverified).toEqual([ - { path: "models/a.sql", reason: "review failed for this file: EACCES: permission denied" }, + { path: "models/a.sql", reason: "review failed for this file: EACCES: permission denied", kind: "failed" }, ]) + // A file that was never reviewed is not a clean pass. + expect(env.verdict).not.toBe("APPROVE") + expect(env.idealVerdict).toBe("COMMENT") }) test("a lane that throws on one file keeps the other file's findings", async () => { @@ -135,7 +138,7 @@ describe("a file whose review failed is never approved", () => { throw new Error("git show failed") }, }) - expect(env.scope?.unverified).toEqual([{ path: "models/schema.yml", reason: "review failed for this file: git show failed" }]) + expect(env.scope?.unverified).toEqual([{ path: "models/schema.yml", reason: "review failed for this file: git show failed", kind: "failed" }]) expect(env.idealVerdict).toBe("COMMENT") }) }) @@ -155,7 +158,11 @@ describe("PII abstention", () => { getContent: async (_f, side) => (side === "new" ? "select email from users" : "select 1 as id"), }) expect(env.findings.filter((f) => f.category === "pii_exposure")).toHaveLength(0) - expect(env.scope?.unverified).toContainEqual({ path: "models/a.sql", reason: "the PII check could not analyse the base model" }) + expect(env.scope?.unverified).toContainEqual({ + path: "models/a.sql", + reason: "the PII check could not analyse the base model", + kind: "abstained", + }) }) test("a PII check that could not analyse the model is reported, not treated as clean", async () => { @@ -171,8 +178,12 @@ describe("PII abstention", () => { }), getContent: async () => "select 1 as id", }) - expect(env.scope?.unverified).toEqual([{ path: "models/a.sql", reason: "the PII check could not analyse this model" }]) + expect(env.scope?.unverified).toEqual([ + { path: "models/a.sql", reason: "the PII check could not analyse this model", kind: "abstained" }, + ]) expect(renderSummary(env)).toContain("the PII check could not analyse this model") + // An abstention is "not checked", so the run cannot APPROVE. + expect(env.idealVerdict).toBe("COMMENT") }) }) @@ -288,6 +299,78 @@ describe("posting: dedup, dismissal of a stale block, no repeat reviews", () => expect(b.calls.dismissReview).toHaveLength(0) }) + test("a rerun where the PII check abstained does not clear an earlier PII block", async () => { + const blocking = [{ id: 11, body: "🛑 Changes requested — 1 critical (full tier)", state: "CHANGES_REQUESTED", user: bot }] + const abstained = buildEnvelope({ + findings: [], + tier: "full", + mode: "gate", + scope: { unverified: [{ path: "models/m.sql", reason: "the PII check could not analyse this model", kind: "abstained" }] }, + }) + expect(abstained.idealVerdict).toBe("COMMENT") + const { client, calls } = fakeClient({ reviews: blocking }) + await postGitHubReview(abstained, target, client) + expect(calls.dismissReview).toHaveLength(0) + }) + + test("a run on an older push never clears the block", async () => { + const env = buildEnvelope({ findings: [], tier: "lite", mode: "gate", scope: { headSha: "old" } }) + const { client, calls } = fakeClient({ + reviews: [{ id: 11, body: "🛑 Changes requested — 1 critical (full tier)", state: "CHANGES_REQUESTED", user: bot }], + }) + await postGitHubReview(env, target, client) + expect(calls.dismissReview).toHaveLength(0) + const current = buildEnvelope({ findings: [], tier: "lite", mode: "gate", scope: { headSha: "abc" } }) + const b = fakeClient({ + reviews: [{ id: 11, body: "🛑 Changes requested — 1 critical (full tier)", state: "CHANGES_REQUESTED", user: bot }], + }) + await postGitHubReview(current, target, b.client) + expect(b.calls.dismissReview.map((c) => c.review_id)).toEqual([11]) + }) + + test("in a monorepo one project's clean run never dismisses another project's block", async () => { + const env = buildEnvelope({ findings: [engineFinding(3)], tier: "lite", mode: "gate", scope: { projectRoot: "projects/a" } }) + const { client, calls } = fakeClient({ + reviews: [ + { + id: 31, + body: "🛑 Changes requested — 1 critical (full tier)\n\n", + state: "CHANGES_REQUESTED", + user: bot, + }, + { + id: 32, + body: "🛑 Changes requested — 1 critical (full tier)\n\n", + state: "CHANGES_REQUESTED", + user: bot, + }, + ], + }) + await postGitHubReview(env, target, client) + expect(calls.dismissReview.map((c) => c.review_id)).toEqual([32]) + expect(calls.createReview[0].body).toContain("") + }) + + test("the block is dismissed only after the replacement review is posted", async () => { + const env = buildEnvelope({ findings: [engineFinding(3)], tier: "lite", mode: "gate" }) + const { client, calls } = fakeClient({ + reviews: [{ id: 11, body: "🛑 Changes requested — 1 critical (full tier)", state: "CHANGES_REQUESTED", user: bot }], + }) + const order: string[] = [] + ;(client.rest.pulls as any).createReview = async () => { + order.push("create") + throw Object.assign(new Error("Server Error"), { status: 502 }) + } + ;(client.rest.pulls as any).dismissReview = async () => { + order.push("dismiss") + return { data: {} } + } + const r = await postGitHubReview(env, target, client) + expect(r.postError).toContain("502") + expect(order).toEqual(["create", "create"]) + expect(calls.dismissReview).toHaveLength(0) + }) + test("a still-blocking rerun does not dismiss and posts REQUEST_CHANGES", async () => { const blocking = makeFinding({ severity: "critical", diff --git a/packages/opencode/test/altimate/review-runner.test.ts b/packages/opencode/test/altimate/review-runner.test.ts index de07bc6ddc..40f832804b 100644 --- a/packages/opencode/test/altimate/review-runner.test.ts +++ b/packages/opencode/test/altimate/review-runner.test.ts @@ -175,7 +175,9 @@ describe("engine handlers that report failure instead of throwing", () => { expect((await runner.check("select 1 as id", "duckdb")).ran).toBe(false) expect((await runner.detectPii("select 1 as id", "duckdb")).abstained).toBe(true) expect(await runner.grain!("select 1 as id")).toEqual({ group_by: [], dedup_partition: [] }) + expect(await runner.dataDiff!("select 1 as id", "select 2 as id", ["id"])).toBeNull() const failed = runner.failures!().map((f) => f.method) + expect(failed).toContain("dataDiff") expect(failed).toContain("check") expect(failed).toContain("grain") expect(runner.failures!()[0].reason).toBe("Error: native core not loaded") diff --git a/packages/opencode/test/altimate/review-scope-diff.test.ts b/packages/opencode/test/altimate/review-scope-diff.test.ts index 88051f363b..0158213966 100644 --- a/packages/opencode/test/altimate/review-scope-diff.test.ts +++ b/packages/opencode/test/altimate/review-scope-diff.test.ts @@ -463,4 +463,35 @@ describe("a shallow CI checkout without the target branch", () => { expect(resolved.source).toBe("github_env") expect(resolved.sha).toBe(branchPoint) }, 60_000) + test("deepening follows --head (not the checkout) and goes beyond 50 commits", async () => { + const exec = (await import("node:util")).promisify((await import("node:child_process")).execFile) + const fsp = (await import("node:fs")).promises + const os = await import("node:os") + const pth = await import("node:path") + const g = (cwd: string, ...a: string[]) => exec("git", a, { cwd }).then((r) => r.stdout.trim()) + const origin = await fsp.realpath(await fsp.mkdtemp(pth.join(os.tmpdir(), "deepen-origin-"))) + await g(origin, "init", "-q", "-b", "main") + await g(origin, "config", "user.email", "t@t") + await g(origin, "config", "user.name", "t") + await g(origin, "config", "commit.gpgsign", "false") + await g(origin, "config", "uploadpack.allowReachableSHA1InWant", "true") + const commit = async (file: string, msg: string) => { + await fsp.writeFile(pth.join(origin, file), `${msg}\n`) + await g(origin, "add", "-A") + await g(origin, "commit", "-qm", msg) + } + await commit("seed.sql", "seed") + const branchPoint = await g(origin, "rev-parse", "HEAD") + for (let i = 0; i < 60; i++) await commit("main.sql", `main ${i}`) + await g(origin, "checkout", "-q", "-b", "pr", branchPoint) + for (let i = 0; i < 60; i++) await commit("pr.sql", `pr ${i}`) + await g(origin, "checkout", "-q", "-b", "other", "main") + await commit("other.sql", "other") + // The checkout is `other`; the review targets the `pr` head. + const clone = await fsp.realpath(await fsp.mkdtemp(pth.join(os.tmpdir(), "deepen-clone-"))) + await exec("git", ["clone", "-q", "--depth", "1", "--branch", "other", `file://${origin}`, clone]) + await g(clone, "fetch", "-q", "--depth", "1", "origin", "+refs/heads/pr:refs/remotes/origin/pr") + const resolved = await resolveReviewBase({ cwd: clone, head: "origin/pr", env: { GITHUB_BASE_REF: "main" } }) + expect(resolved.sha).toBe(branchPoint) + }, 60_000) })