Repository navigation
feat(review): opt-in AI verifier that confirms or refutes each AI comment from dbt evidence #1769
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: feat/review-posting-policy
Are you sure you want to change the base?
Changes from all commits
8d5c12d
e61b176
64eacef
116512c
c9fd468
a9dd602
8399cad
09b340a
4d89d6e
42fb3fb
3b2c31e
933b0a2
f3c5e86
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,203 @@ | ||
| // altimate_change - new file: replay the AI lane's generator findings from an earlier envelope | ||
| // | ||
| // `ALTIMATE_REVIEW_AI_REPLAY=<path to a review --json envelope>` skips the | ||
| // generator model call and rebuilds its findings from that envelope: the AI | ||
| // findings it kept (with what the verifier added undone) plus the ones the | ||
| // verifier discarded (`ai.verifier.dropped`). The verifier then runs on them as | ||
| // usual, so verifier settings can be compared without paying for generation | ||
| // again. Ids are the stored ones; benchmark labels are keyed by them. | ||
| import { readFile } from "node:fs/promises" | ||
| import z from "zod" | ||
| import { Provider } from "@/provider/provider" | ||
| import { Log } from "@/altimate/util/log" | ||
| import { Finding, ReviewCategory, makeFinding, ruleId } from "./finding" | ||
| import type { AiLaneStatus, AiReviewInput, AiReviewResult } from "./ai-review" | ||
| import { aiVerifyEnabled, runAiVerify, type VerifierStatus } from "./ai-verify" | ||
| import { modelNameFromPath } from "./orchestrate" | ||
|
|
||
| const log = Log.create({ service: "ai-replay" }) | ||
|
|
||
| /** The path of the envelope to replay, when replay is on. */ | ||
| export function aiReplayPath(env: Record<string, string | undefined> = process.env): string | undefined { | ||
| return env.ALTIMATE_REVIEW_AI_REPLAY || undefined | ||
| } | ||
|
|
||
| const Tokens = z.object({ | ||
| input: z.number(), | ||
| inputTotal: z.number(), | ||
| output: z.number(), | ||
| reasoning: z.number(), | ||
| cacheRead: z.number(), | ||
| }) | ||
|
|
||
| /** The parts of an envelope replay reads; everything else is ignored. */ | ||
| const ReplayEnvelope = z.object({ | ||
| findings: z.array(z.unknown()), | ||
| ai: z | ||
| .object({ | ||
| state: z.enum(["ok", "skipped", "timeout", "error"]), | ||
| reason: z.string().optional(), | ||
| model: z.string().optional(), | ||
| tokens: Tokens.optional(), | ||
| cost: z.number().optional(), | ||
| calls: z.number().optional(), | ||
| verifier: z | ||
| .object({ | ||
| discarded: z.number().optional(), | ||
| dropped: z | ||
| .array( | ||
| z.object({ | ||
| id: z.string(), | ||
| file: z.string(), | ||
| startLine: z.number().optional(), | ||
| title: z.string(), | ||
| body: z.string(), | ||
| category: z.string(), | ||
| severity: z.string(), | ||
| }), | ||
| ) | ||
| .optional(), | ||
| decisions: z | ||
| .array(z.object({ id: z.string(), ts: z.object({ lowConfidence: z.boolean() }).partial().optional() })) | ||
| .optional(), | ||
| }) | ||
| .optional(), | ||
| }) | ||
| .optional(), | ||
| }) | ||
|
|
||
| export interface ReplayedLane { | ||
| findings: Finding[] | ||
| /** The stored generator status: state, reason, model, tokens, cost, calls. */ | ||
| status: Omit<AiLaneStatus, "durationMs" | "verifier"> | ||
| /** Problems found while rebuilding (findings that could not be recovered). */ | ||
| warnings: string[] | ||
| } | ||
|
|
||
| /** | ||
| * Rebuild the generator's findings from a parsed envelope (pure). Kept AI | ||
| * findings lose the verifier's record, its `Verified:` body suffix and the | ||
| * tiers set after generation (the generator sets none); discarded ones are | ||
| * rebuilt from `ai.verifier.dropped`. Order follows the verifier's | ||
| * `decisions` (the generator's order) when recorded. | ||
| */ | ||
| export function replayFindings(raw: unknown): ReplayedLane { | ||
| const env = ReplayEnvelope.parse(raw) | ||
| const ai = env.ai | ||
| if (!ai) throw new Error("the envelope has no AI lane status") | ||
| const status: ReplayedLane["status"] = { | ||
| state: ai.state, | ||
| ...(ai.reason ? { reason: ai.reason } : {}), | ||
| ...(ai.model ? { model: ai.model } : {}), | ||
| ...(ai.tokens ? { tokens: ai.tokens } : {}), | ||
| ...(ai.cost !== undefined ? { cost: ai.cost } : {}), | ||
| ...(ai.calls !== undefined ? { calls: ai.calls } : {}), | ||
| replayed: true, | ||
| } | ||
| const warnings: string[] = [] | ||
| const out: Finding[] = [] | ||
| for (const item of env.findings) { | ||
| const parsed = Finding.safeParse(item) | ||
| if (!parsed.success || parsed.data.evidence?.tool !== "ai-review") continue | ||
| out.push(unverify(parsed.data)) | ||
| } | ||
|
|
||
| const v = ai.verifier | ||
| const lowConfidence = new Map((v?.decisions ?? []).map((d) => [d.id, d.ts?.lowConfidence === true])) | ||
| for (const d of v?.dropped ?? []) { | ||
| const category = ReviewCategory.safeParse(d.category) | ||
| if (!category.success) { | ||
| warnings.push(`discarded finding ${d.id} has an unknown category "${d.category}"`) | ||
| continue | ||
| } | ||
| // The envelope does not keep a discarded finding's confidence; the verifier reads only whether it is low. | ||
| const confidence = lowConfidence.get(d.id) ? ("low" as const) : ("medium" as const) | ||
| out.push( | ||
| makeFinding({ | ||
| id: d.id, | ||
| severity: d.severity === "warning" ? "warning" : "suggestion", | ||
| category: category.data, | ||
| title: d.title, | ||
| body: d.body, | ||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. P3: Discarded findings are rebuilt from the envelope's Prompt for AI agents |
||
| file: d.file, | ||
| model: modelNameFromPath(d.file), | ||
| ...(d.startLine ? { startLine: d.startLine, endLine: d.startLine } : {}), | ||
| confidence, | ||
| evidence: { tool: "ai-review", result: { confidence } }, | ||
| rule: ruleId("ai-review", d.category), | ||
| }), | ||
| ) | ||
| } | ||
|
|
||
| const listed = v?.dropped?.length ?? 0 | ||
| if ((v?.discarded ?? 0) > listed) | ||
| warnings.push(`the verifier discarded ${v!.discarded} findings but the envelope lists ${listed}`) | ||
| const have = new Set(out.map((f) => f.id)) | ||
| const missing = (v?.decisions ?? []).filter((d) => !have.has(d.id)).length | ||
| if (missing) warnings.push(`${missing} findings the verifier decided are not in the envelope`) | ||
|
|
||
| if (v?.decisions?.length) { | ||
| const order = new Map(v.decisions.map((d, i) => [d.id, i])) | ||
| out.sort((a, b) => (order.get(a.id) ?? Infinity) - (order.get(b.id) ?? Infinity)) | ||
| } | ||
| return { findings: out, status: warnings.length ? { ...status, replayIncomplete: true } : status, warnings } | ||
| } | ||
|
|
||
| /** A kept finding as the generator returned it: no verification record, body suffix, tier or origin. */ | ||
| function unverify(f: Finding): Finding { | ||
| const { tier: _tier, origin: _origin, ...rest } = f | ||
| const prior = f.evidence?.result | ||
| if (!prior || typeof prior !== "object") return rest | ||
| const { verification, ...result } = prior as Record<string, unknown> & { | ||
| verification?: { rule?: string; evidence?: string } | ||
| } | ||
| let body = f.body | ||
| const suffix = verification?.rule === "verified" && verification.evidence ? `\n\nVerified: ${verification.evidence}` : "" | ||
| if (suffix && body.endsWith(suffix)) body = body.slice(0, -suffix.length) | ||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. SUGGESTION:
Remove the last occurrence of the suffix instead of requiring a tail match, and strip the |
||
| return { ...rest, body, evidence: { tool: "ai-review", result } } | ||
| } | ||
|
|
||
| /** | ||
| * The AI lane in replay mode: no generator call. Fails loudly (stderr) and | ||
| * reports `state: "error"` when the envelope cannot be read; never falls back | ||
| * to calling the generator. | ||
| */ | ||
| export async function runAiReplay(input: AiReviewInput, path: string): Promise<AiReviewResult> { | ||
| const startedAt = Date.now() | ||
| const done = (findings: Finding[], status: Omit<AiLaneStatus, "durationMs">): AiReviewResult => ({ | ||
| findings, | ||
| status: { ...status, durationMs: Date.now() - startedAt }, | ||
| }) | ||
| let lane: ReplayedLane | ||
| try { | ||
| lane = replayFindings(JSON.parse(await readFile(path, "utf8"))) | ||
| } catch (err) { | ||
| const reason = `cannot replay ${path}: ${err instanceof Error ? err.message : String(err)}`.slice(0, 300) | ||
| process.stderr.write(`altimate review: ALTIMATE_REVIEW_AI_REPLAY: ${reason}\n`) | ||
| log.error("ai replay failed", { path, error: err }) | ||
| return done([], { state: "error", reason, replayed: true }) | ||
| } | ||
| for (const w of lane.warnings) { | ||
| process.stderr.write(`altimate review: ALTIMATE_REVIEW_AI_REPLAY: ${w}\n`) | ||
| log.warn("ai replay incomplete", { path, warning: w }) | ||
| } | ||
| log.info("ai replay", { path, findings: lane.findings.length }) | ||
| if (lane.status.state !== "ok") return done([], lane.status) | ||
|
|
||
| let findings = lane.findings | ||
| let verifier: VerifierStatus | undefined | ||
| if (input.evidence && aiVerifyEnabled()) { | ||
| let model: Awaited<ReturnType<typeof Provider.getModel>> | ||
| try { | ||
| const chosen = input.model ? Provider.parseModel(input.model) : await Provider.defaultModel() | ||
| model = await Provider.getModel(chosen.providerID, chosen.modelID) | ||
| } catch (err) { | ||
| const reason = `no usable model: ${err instanceof Error ? err.message : String(err)}`.slice(0, 200) | ||
| return done([], { ...lane.status, state: "skipped", reason }) | ||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. P1: When replay cannot resolve a verifier model, this branch drops every stored AI finding even though verification never ran. Return Prompt for AI agents
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. [MINOR] Replay drops all replayed findings when no verifier model resolves return done([], { ...lane.status, state: "skipped", reason })The generator findings are already rebuilt at this point, but are discarded on a model-lookup failure, contradicting the module's fail-open design. Scenario: Fix: return |
||
| } | ||
| const v = await runAiVerify({ findings, grounding: input.grounding, evidence: input.evidence, model }) | ||
| findings = v.findings | ||
| verifier = v.status | ||
| } | ||
| return done(findings, { ...lane.status, ...(verifier ? { verifier } : {}) }) | ||
| } | ||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -9,6 +9,8 @@ import { redactSecrets } from "@/installation" | |
| import { Dispatcher } from "../native" | ||
| import { type Finding, type ReviewCategory, type Severity, makeFinding, ruleId } from "./finding" | ||
| import { readStreamText } from "../llm/stream-text" | ||
| import { type AiVerifyEvidence, type VerifierStatus, aiVerifyEnabled, runAiVerify } from "./ai-verify" | ||
| import { aiReplayPath, runAiReplay } from "./ai-replay" | ||
|
|
||
| const log = Log.create({ service: "ai-review" }) | ||
|
|
||
|
|
@@ -138,6 +140,8 @@ export interface AiReviewInput { | |
| prBody?: string | ||
| /** Explicit `provider/model`; omit to use the configured default model. */ | ||
| model?: string | ||
| /** What the verifier may read; it runs only when this is set and `ALTIMATE_REVIEW_AI_VERIFY=1`. */ | ||
| evidence?: AiVerifyEvidence | ||
| } | ||
|
|
||
| /** What happened in the advisory lane, reported on the envelope and in the comment. */ | ||
|
|
@@ -155,6 +159,13 @@ export interface AiLaneStatus { | |
| cost?: number | ||
| /** Model calls made (one per batch). */ | ||
| calls?: number | ||
| /** The dbt-grounded verifier's outcome, when it ran (opt-in). */ | ||
| verifier?: VerifierStatus | ||
| /** The generator's findings came from an earlier envelope (`ALTIMATE_REVIEW_AI_REPLAY`); | ||
| * tokens, cost and calls are that run's, so only `verifier` was spent now. */ | ||
| replayed?: boolean | ||
| /** The replayed envelope did not list every generator finding. */ | ||
| replayIncomplete?: boolean | ||
| } | ||
| /** Token counts as `Session.getUsage` reports them: `input` excludes cached | ||
| * tokens, `inputTotal` includes them. */ | ||
|
|
@@ -253,6 +264,8 @@ export function buildUserMessage(input: AiReviewInput, maxFileChars = MAX_DIFF_C | |
| * envelope and the PR comment. | ||
| */ | ||
| export async function runAiReview(input: AiReviewInput): Promise<AiReviewResult> { | ||
| const replay = aiReplayPath() | ||
| if (replay) return runAiReplay(input, replay) | ||
| const startedAt = Date.now() | ||
| const done = (findings: Finding[], status: Omit<AiLaneStatus, "durationMs">): AiReviewResult => ({ | ||
| findings, | ||
|
|
@@ -291,15 +304,22 @@ export async function runAiReview(input: AiReviewInput): Promise<AiReviewResult> | |
| const slice = batches.slice(i, i + BATCH_CONCURRENCY) | ||
| results.push(...(await Promise.all(slice.map((b) => reviewBatch(input, b, budget, system, model))))) | ||
| } | ||
| const findings = results.flatMap((r) => r.findings) | ||
| let findings = results.flatMap((r) => r.findings) | ||
| const failed = results.filter((r) => r.state !== "ok") | ||
| log.info("ai review complete", { findings: findings.length, batches: batches.length, failed: failed.length }) | ||
| // The lane is ok when any call answered; a partial failure is reported, not hidden. | ||
| const spent = laneUsage(results) | ||
| if (failed.length === results.length) | ||
| return done([], { state: failed[0].state, reason: failed[0].reason, model: modelLabel, ...spent }) | ||
| const reason = failed.length ? `${failed.length} of ${results.length} calls failed: ${failed[0].reason}` : undefined | ||
| return done(findings, { state: "ok", model: modelLabel, ...(reason ? { reason } : {}), ...spent }) | ||
| // Opt-in: try to refute each finding with dbt evidence. Never throws. | ||
| let verifier: VerifierStatus | undefined | ||
| if (input.evidence && aiVerifyEnabled()) { | ||
| const v = await runAiVerify({ findings, grounding: input.grounding, evidence: input.evidence, model }) | ||
| findings = v.findings | ||
| verifier = v.status | ||
| } | ||
| return done(findings, { state: "ok", model: modelLabel, ...(reason ? { reason } : {}), ...spent, ...(verifier ? { verifier } : {}) }) | ||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. P2: With the verifier on, lane-level Prompt for AI agents |
||
| } catch (err) { | ||
| log.error("ai review failed", { error: err }) | ||
| return done([], { state: "error", reason: safeReason(err), model: modelLabel }) | ||
|
|
||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
P3:
readStreamStepadds a new error-propagation path and aninvalidpassthrough, but no test covers either. Add cases to stream-text.test.ts for a mid-streamerrorpart (must throw) and for a tool call withinvalid: true.Prompt for AI agents