diff --git a/.github/ARCHITECTURE.md b/.github/ARCHITECTURE.md index 03e12f0..85c671c 100644 --- a/.github/ARCHITECTURE.md +++ b/.github/ARCHITECTURE.md @@ -17,6 +17,7 @@ The result is a draft pull request. Merge and ready-for-review actions are inten ## Eve capabilities - `agent/channels/github.ts` handles authorized mentions, `factory`-label intake, CI-failure follow-up, and PR summaries. +- Failure comments are policy-controlled. `lib/failure-policy.ts` decides what a channel may say when a turn or session fails: a **deployment fault** (an unusable credential, an unpaid gateway account, or a model the account cannot reach) posts nothing, because only an operator can clear it, and every other failure posts one generic sentence that never carries the upstream provider's text. eve's built-in handler echoes that text verbatim, which is how a revoked key repeated `Model provider API error: Authentication Fails, Your api key: ****53a6...` into the originating thread on every dispatch. - `agent/channels/discord-mentions.ts` owns ordinary `@Computer` mentions in the internal Discord channel; `bridge/discord-gateway/` holds the Gateway connection that carries them, and `lib/discord-mention-policy.ts` decides admission. - `agent/extensions/github.ts` mounts the official GitHub tools with an explicit allowlist and the WazooComputer GitHub App installation token. - `agent/subagents/` contains isolated station prompts, sandboxes, and handoff tools. diff --git a/agent/channels/discord-mentions.ts b/agent/channels/discord-mentions.ts index ef4891b..c324c73 100644 --- a/agent/channels/discord-mentions.ts +++ b/agent/channels/discord-mentions.ts @@ -21,6 +21,7 @@ import { createDedupeCache, verifyDiscordBridgeRequest, } from "../../lib/discord-bridge.ts"; +import { describeFailure, failureCommentBody } from "../../lib/failure-policy.ts"; import { discordPolicyConfigFromEnv } from "../../lib/discord-policy.ts"; import { type DiscordMentionAdmission, @@ -145,6 +146,27 @@ function ignored(reason: string): Response { return Response.json({ dispatched: false, ok: true, reason }, { status: 202 }); } +/** + * Posts the failure notice for one failed turn or session, or nothing at all + * when the failure is a deployment fault (see `lib/failure-policy.ts`); the + * evidence goes to the runtime log instead of the channel. + */ +async function postFailureNotice( + failure: { + readonly code: string; + readonly details?: Record | undefined; + readonly message: string; + }, + channel: DiscordMentionContext, +): Promise { + const body = failureCommentBody(failure); + if (body === null) { + console.error("computer deployment fault suppressed on Discord", describeFailure(failure)); + return; + } + await tryPost(channel.state, body); +} + /** Forwards one admitted mention into its channel-scoped session. */ async function dispatchMention(from: ChannelFrom, payload: unknown): Promise { if (typeof payload !== "object" || payload === null) return ignored("malformed-payload"); @@ -281,11 +303,11 @@ export default defineChannel({ if (typeof data.message !== "string" || data.message.length === 0) return; await tryPost(channel.state, data.message); }, - async "session.failed"(_data, channel) { - await tryPost(channel.state, "Computer could not recover from an error on that request. Please try again."); + async "session.failed"(data, channel) { + await postFailureNotice(data, channel); }, - async "turn.failed"(_data, channel) { - await tryPost(channel.state, "Computer hit an error handling that request. Please try again or rephrase it."); + async "turn.failed"(data, channel) { + await postFailureNotice(data, channel); }, async "turn.started"(_data, channel) { await tryTyping(channel.state); diff --git a/agent/channels/github.ts b/agent/channels/github.ts index 0f8988c..e5328df 100644 --- a/agent/channels/github.ts +++ b/agent/channels/github.ts @@ -1,6 +1,7 @@ import { defaultGitHubAuth, type GitHubComment, + type GitHubEventContext, type GitHubInboundContext, githubChannel, } from "eve/channels/github"; @@ -33,6 +34,7 @@ import { repositoryTargetFromInbound, } from "../lib/github/repository-target.js"; import { intakeStateForLabels, planIntakeStateTransition } from "../lib/intake-policy.js"; +import { describeFailure, failureCommentBody } from "../../lib/failure-policy.js"; const githubCredentials = { appId: () => process.env.GITHUB_APP_ID ?? "", @@ -262,6 +264,34 @@ const PR_SUMMARY_TASK = [ const BODY_MENTION_ACTIONS = new Set(["opened", "edited"]); +/** + * Posts the failure comment for one failed turn or session, or nothing at all + * when the failure is a deployment fault (see `lib/failure-policy.ts`). + * + * The deployment evidence goes to the runtime log instead of the thread: a + * revoked credential is an operator problem, and repeating it in a reply is + * noise on somebody else's conversation. + */ +async function postFailureComment( + failure: { + readonly code: string; + readonly details?: Record | undefined; + readonly message: string; + }, + channel: GitHubEventContext, +): Promise { + const body = failureCommentBody(failure); + if (body === null) { + console.error("computer deployment fault suppressed on GitHub", describeFailure(failure)); + return; + } + try { + await channel.thread.post(body); + } catch (error) { + console.error("github failure comment could not be posted", { error }); + } +} + /** * GitHub channel: the factory's main intake and delivery surface, as * "Computer". @@ -322,6 +352,17 @@ const BODY_MENTION_ACTIONS = new Set(["opened", "edited"]); export default githubChannel({ botName: resolveTeamMention, credentials: githubCredentials, + // Only the two failure events are overridden: eve's defaults post the + // provider's own words for a model rejection, and `turn.started` owns the + // repository checkout, so it must stay the framework's. + events: { + async "turn.failed"(data, channel) { + await postFailureComment(data, channel); + }, + async "session.failed"(data, channel) { + await postFailureComment(data, channel); + }, + }, onCheckSuite: async (ctx, suite) => { const raw = suite.raw as { head_branch?: unknown; diff --git a/lib/failure-policy.test.ts b/lib/failure-policy.test.ts new file mode 100644 index 0000000..af1e093 --- /dev/null +++ b/lib/failure-policy.test.ts @@ -0,0 +1,99 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import { + describeFailure, + failureCommentBody, + isDeploymentFault, +} from "./failure-policy.ts"; + +/** + * The payload eve attached on the 2026-09-21 dispatches that took Computer + * offline: a revoked DeepSeek key, surfaced as a provider rejection with the + * provider's own message in the event. + */ +const revokedKeyFailure = { + code: "turn_failed", + details: { + errorId: "5f2c1d0e-9a3b-4c7d-8e1f-2b3c4d5e6f70", + name: "Model provider API error", + statusCode: 401, + upstreamType: "authentication_error", + }, + message: "Model provider API error: Authentication Fails, Your api key: ****53a6 is invalid", +}; + +test("suppresses the provider rejection that takes the deployment offline", () => { + assert.equal(isDeploymentFault(revokedKeyFailure), true); + assert.equal(failureCommentBody(revokedKeyFailure), null); +}); + +test("suppresses a gateway rejection of a model the account cannot reach", () => { + const failure = { + code: "turn_failed", + details: { + name: "AI Gateway model request rejected", + statusCode: 403, + upstreamType: "model_not_found", + }, + message: "AI Gateway rejected the model request before the agent produced a response.", + }; + assert.equal(isDeploymentFault(failure), true); + assert.equal(failureCommentBody(failure), null); +}); + +test("treats a bare credential status code as a deployment fault", () => { + const failure = { + code: "session_failed", + details: { statusCode: 402 }, + message: "upstream request failed", + }; + assert.equal(isDeploymentFault(failure), true); + assert.equal(failureCommentBody(failure), null); +}); + +test("treats a rejection named only inside the message as a deployment fault", () => { + const failure = { + code: "turn_failed", + details: {}, + message: "Model provider API error: Authentication Fails, Your api key is invalid", + }; + assert.equal(isDeploymentFault(failure), true); + assert.equal(failureCommentBody(failure), null); +}); + +test("still answers an ordinary failure, with the correlation id and no provider text", () => { + const failure = { + code: "turn_failed", + details: { errorId: "9d1b2c3a-4e5f-4061-8273-8495a6b7c8d9", statusCode: 400 }, + message: "Tool input did not match the declared schema", + }; + const body = failureCommentBody(failure); + assert.ok(body !== null); + assert.match(body, /could not recover/u); + assert.match(body, /Error id: 9d1b2c3a-4e5f-4061-8273-8495a6b7c8d9/u); + assert.doesNotMatch(body, /Tool input did not match the declared schema/u); +}); + +test("answers an ordinary failure that carries no correlation id", () => { + const failure = { code: "turn_failed", details: {}, message: "sandbox checkout timed out" }; + const body = failureCommentBody(failure); + assert.ok(body !== null); + assert.doesNotMatch(body, /Error id/u); + assert.doesNotMatch(body, /sandbox checkout timed out/u); +}); + +test("describes a failure from structured fields only", () => { + const description = describeFailure(revokedKeyFailure); + assert.equal( + description, + "turn_failed | Model provider API error | upstream authentication_error | HTTP 401 | error id 5f2c1d0e-9a3b-4c7d-8e1f-2b3c4d5e6f70", + ); + assert.doesNotMatch(description, /\*\*\*53a6/u); +}); + +test("survives a failure with no details at all", () => { + const failure = { code: "session_failed", message: "" }; + assert.equal(isDeploymentFault(failure), false); + assert.ok(failureCommentBody(failure) !== null); + assert.equal(describeFailure(failure), "session_failed"); +}); diff --git a/lib/failure-policy.ts b/lib/failure-policy.ts new file mode 100644 index 0000000..76a93db --- /dev/null +++ b/lib/failure-policy.ts @@ -0,0 +1,114 @@ +/** + * Failure-comment policy for Computer's chat channels. + * + * eve's built-in channel handlers post `details.name` plus the raw failed-event + * message. For a model-provider rejection that message is the upstream + * provider's own words, which is how a revoked key put + * "Model provider API error: Authentication Fails, Your api key: ****53a6…" + * into the originating thread on every dispatch (#77). + * + * Two rules follow from that: + * + * - A **deployment fault** — an unusable credential, an unpaid gateway + * account, or a model this account cannot reach — is not a reply to the + * person who asked. Only an operator can clear it, so the channel stays + * silent and the evidence goes to the runtime log instead of the thread. + * - **No failure comment carries provider text.** Every other failure gets one + * short reply, with the correlation id when the event has one, so a support + * request can still be traced back to one incident. + * + * The provider's own message is never interpolated anywhere in this module, so + * neither the comment nor the log line can leak a credential. + */ + +export type FailureEvent = { + readonly code: string; + readonly details?: Record | undefined; + readonly message: string; +}; + +/** `details.name` values eve sets when the model call was rejected upstream. */ +const PROVIDER_REJECTION_NAMES = new Set([ + "AI Gateway model request rejected", + "Model provider API error", +]); + +/** Gateway `upstreamType` values that no retry can clear. */ +const TERMINAL_UPSTREAM_TYPES = new Set([ + "authentication_error", + "invalid_request_error", + "model_not_found", +]); + +/** HTTP statuses that name the deployment, not the request, as the fault. */ +const DEPLOYMENT_STATUS_CODES = new Set([401, 402, 403]); + +/** Matches the rejection name when a cascade carries it only in `message`. */ +const PROVIDER_REJECTION_PATTERN = /\b(?:AI Gateway model request rejected|Model provider API error)\b/u; + +const readString = (value: unknown): string | undefined => + typeof value === "string" && value.trim().length > 0 ? value : undefined; + +const readNumber = (value: unknown): number | undefined => + typeof value === "number" && Number.isFinite(value) ? value : undefined; + +/** + * Reads the correlation id eve attaches to a failed event, so a support ticket + * quoting the id can be grepped back to one incident. + */ +export function failureErrorId(details: unknown): string | undefined { + if (typeof details !== "object" || details === null) return undefined; + return readString((details as { errorId?: unknown }).errorId); +} + +/** + * True when the failure is a deployment fault that only an operator can clear. + * + * Signals are checked in specificity order: the structured rejection name, the + * gateway's own terminal `upstreamType`, the status code, then the name inside + * `message` for older cascades that predate `details.name`. + */ +export function isDeploymentFault(failure: FailureEvent): boolean { + const details = failure.details ?? {}; + const name = readString(details.name); + if (name !== undefined && PROVIDER_REJECTION_NAMES.has(name)) return true; + const upstreamType = readString(details.upstreamType); + if (upstreamType !== undefined && TERMINAL_UPSTREAM_TYPES.has(upstreamType)) return true; + const statusCode = readNumber(details.statusCode); + if (statusCode !== undefined && DEPLOYMENT_STATUS_CODES.has(statusCode)) return true; + return name === undefined && PROVIDER_REJECTION_PATTERN.test(failure.message); +} + +/** + * The comment a channel posts for a failed turn or session, or `null` when the + * channel should stay silent because only an operator can clear the failure. + */ +export function failureCommentBody(failure: FailureEvent): string | null { + if (isDeploymentFault(failure)) return null; + const id = failureErrorId(failure.details); + return [ + "I hit an error while handling that request and could not recover.", + "", + "Please try again or rephrase it.", + ...(id === undefined ? [] : ["", `Error id: ${id}`]), + ].join("\n"); +} + +/** + * A one-line summary for the runtime log, built from structured fields only. + * The provider's message is deliberately absent: the point of logging here is + * that the thread stays quiet, not that the text moves somewhere else. + */ +export function describeFailure(failure: FailureEvent): string { + const details = failure.details ?? {}; + const parts = [failure.code]; + const name = readString(details.name); + if (name !== undefined) parts.push(name); + const upstreamType = readString(details.upstreamType); + if (upstreamType !== undefined) parts.push(`upstream ${upstreamType}`); + const statusCode = readNumber(details.statusCode); + if (statusCode !== undefined) parts.push(`HTTP ${statusCode}`); + const id = failureErrorId(details); + if (id !== undefined) parts.push(`error id ${id}`); + return parts.join(" | "); +}