diff --git a/apps/server/src/orchestration-v2/Adapters/AcpAdapterV2.test.ts b/apps/server/src/orchestration-v2/Adapters/AcpAdapterV2.test.ts index a0aa6ed3130f..f912114d81ce 100644 --- a/apps/server/src/orchestration-v2/Adapters/AcpAdapterV2.test.ts +++ b/apps/server/src/orchestration-v2/Adapters/AcpAdapterV2.test.ts @@ -1527,6 +1527,117 @@ describe("AcpAdapterV2", () => { ); }).pipe(Effect.provide(layerTest)), ); + it.live("resends a resumable outcome failure's prompt once before failing the turn", () => + Effect.gen(function* () { + const instanceId = ProviderInstanceId.make("acp-test-prompt-resume"); + const threadId = ThreadId.make("thread-acp-prompt-resume"); + const notFound = { + class: "provider_error", + message: "Requested entity was not found.", + code: "NOT_FOUND", + retryable: true, + } as const; + const promptTexts: Array = []; + // Checks 1, 3 and 4 fail: turn 1 resumes and completes, turn 2 fails after one resume. + let checks = 0; + const adapter = yield* makeAcpAdapterV2({ + instanceId, + selfInvocation: yield* resolveSelfInvocation(), + flavor: { + driver: ACP_TEST_DRIVER, + capabilities: AcpProviderCapabilitiesV2, + steersWithPrompt: () => true, + watchPromptOutcome: () => + Effect.sync(() => { + checks += 1; + const outcome = + checks === 2 + ? undefined + : { + failure: notFound, + threadDisposition: "reusable" as const, + resumeWith: "continue", + }; + return () => Effect.succeed(outcome); + }), + makeRuntime: makeMockRuntime({ + childProcessSpawner: yield* ChildProcessSpawner.ChildProcessSpawner, + mockAgentPath: yield* bundledMockAgentPath, + wrapRuntime: (runtime) => ({ + ...runtime, + prompt: (payload) => + Effect.sync(() => { + promptTexts.push( + payload.prompt + .flatMap((block) => (block.type === "text" ? [block.text] : [])) + .join("\n"), + ); + return { stopReason: "end_turn" as const }; + }), + }), + }), + }, + }); + const runtimePolicy = ProviderAdapter.ProviderAdapterV2RuntimePolicy.make({ + runtimeMode: "full-access", + interactionMode: "default", + cwd: process.cwd(), + }); + const modelSelection = { instanceId, model: "default" } as const; + const runtime = yield* adapter.openSession({ + threadId, + providerSessionId: ProviderSessionId.make("provider-session-acp-prompt-resume"), + modelSelection, + runtimePolicy, + }); + const providerThread = yield* runtime.ensureThread({ + threadId, + modelSelection, + runtimePolicy, + }); + const runTurn = (ordinal: number) => + Effect.gen(function* () { + yield* runtime.startTurn( + makeTurnInput({ + threadId, + providerThread, + instanceId, + runtimePolicy, + now: yield* DateTime.now, + ordinal, + }), + ); + return Array.from( + yield* runtime.events.pipe( + Stream.takeUntil((event) => event.type === "turn.terminal"), + Stream.runCollect, + ), + ); + }); + const retryTitles = (events: ReadonlyArray) => + events.flatMap((event) => + event.type === "turn_item.updated" && + event.turnItem.type === "error" && + event.turnItem.retry !== undefined + ? [event.turnItem.title] + : [], + ); + + const resumed = yield* runTurn(1); + const completed = resumed.at(-1); + assert.equal(completed?.type === "turn.terminal" ? completed.status : null, "completed"); + assert.deepEqual(promptTexts.slice(1), ["continue"]); + assert.deepEqual(retryTitles(resumed), ["Provider retry", "Provider recovered"]); + + const failedEvents = yield* runTurn(2); + const failed = failedEvents.at(-1); + if (failed?.type !== "turn.terminal" || failed.status !== "failed") { + return assert.fail("expected a failed terminal"); + } + assert.deepEqual(failed.failure, notFound); + assert.deepEqual(promptTexts.slice(3), ["continue"], "a turn resumes only once"); + }).pipe(Effect.provide(layerTest)), + ); it.live("retains returned usage when Stop finalizes before the prompt callback", () => Effect.gen(function* () { const instanceId = ProviderInstanceId.make("acp-test-prompt-usage-stop"); diff --git a/apps/server/src/provider/acp/AntigravityTurnOutcome.test.ts b/apps/server/src/provider/acp/AntigravityTurnOutcome.test.ts index eb0695948b30..b6094e638285 100644 --- a/apps/server/src/provider/acp/AntigravityTurnOutcome.test.ts +++ b/apps/server/src/provider/acp/AntigravityTurnOutcome.test.ts @@ -40,6 +40,16 @@ const MISSING_SIGNATURE_ERROR = errorPayload( const INVALID_SIGNATURE_ERROR = errorPayload( 'Error 400, Message: {"type":"error","error":{"type":"invalid_request_error","message":"messages.3.content.0: Invalid `signature` in `thinking` block"}}, Status: INVALID_ARGUMENT, Details: []', ); +const NOT_FOUND_ERROR = errorPayload( + "Error 404, Message: Requested entity was not found., Status: NOT_FOUND, Details: []", +); +// Antigravity 1.5 with a Google account writes "STATUS (code NNN): message", then the HTTP body. +const STATUS_NOT_FOUND_ERROR = Buffer.concat([ + Buffer.from([0x12, 0x37]), + Buffer.from( + 'NOT_FOUND (code 404): Requested entity was not found.\u001a\u0080\u0001HTTP 404 Not Found\n\n{"error":{"code":404,"message":"Requested entity was not found.","status":"NOT_FOUND"}}', + ), +]); // The model's reply quoting the agent's quota notice is an ordinary planner response. const QUOTED_NOTICE = Buffer.from( "Usage Limit Reached\n\nYou have reached your current quota for this period. Your limit will reset in 1 hour.", @@ -139,6 +149,34 @@ describe("watchAntigravityPromptOutcome", () => { retryable: false, }), ); + // A 404 after the turn already reached the model is a transient edge error, not a bad model. + for (const error of [NOT_FOUND_ERROR, STATUS_NOT_FOUND_ERROR]) { + assert.deepEqual( + yield* turn([ + [USER_INPUT, input], + [PLANNER_RESPONSE, reply], + [TOOL, reply], + [ERROR_MESSAGE, error], + ]), + { + ...reusable({ + class: "provider_error", + message: "Requested entity was not found.", + code: "NOT_FOUND", + retryable: true, + }), + resumeWith: "continue", + }, + ); + } + // A 404 before the model answered is an unknown or unentitled model. + assert.equal( + (yield* turn([ + [USER_INPUT, input], + [ERROR_MESSAGE, STATUS_NOT_FOUND_ERROR], + ]))?.failure.retryable, + false, + ); // A saved thinking block without a valid signature fails every later prompt. for (const error of [MISSING_SIGNATURE_ERROR, INVALID_SIGNATURE_ERROR]) { const outcome = yield* turn([ diff --git a/apps/server/src/provider/acp/AntigravityTurnOutcome.ts b/apps/server/src/provider/acp/AntigravityTurnOutcome.ts index 2cbe4d62b3a4..32eee40f008a 100644 --- a/apps/server/src/provider/acp/AntigravityTurnOutcome.ts +++ b/apps/server/src/provider/acp/AntigravityTurnOutcome.ts @@ -15,10 +15,14 @@ import { antigravityManagedPath, antigravityProfileRoot } from "./AntigravityPro // CORTEX_STEP_TYPE_ERROR_MESSAGE in the native trajectory. const ERROR_STEP_TYPE = 17; +// CORTEX_STEP_TYPE_PLANNER_RESPONSE: the model answered within this turn. +const PLANNER_RESPONSE_STEP_TYPE = 15; // The harness error text leads the payload; a Go stack trace follows it. const MAX_ERROR_PAYLOAD_BYTES = 16_384; // The Go harness's rendering of a Google API error, as the agent's api_errors.py reads it. const API_ERROR = /Error (\d{3}), Message: (.+?), Status: ([A-Z_]+)/s; +// Antigravity 1.5 with a Google account writes "STATUS (code NNN): message". +const STATUS_ERROR = /(?()( @@ -90,7 +95,15 @@ const lastStepAfter = ) .get(MAX_ERROR_PAYLOAD_BYTES, index); if (typeof row?.type !== "number") return undefined; - return { type: row.type, payload: row.payload instanceof Uint8Array ? row.payload : null }; + const reachedModel = + database + .prepare("SELECT 1 FROM steps WHERE idx > ? AND step_type = ? LIMIT 1") + .get(index, PLANNER_RESPONSE_STEP_TYPE) !== undefined; + return { + type: row.type, + payload: row.payload instanceof Uint8Array ? row.payload : null, + reachedModel, + }; }; function resetAtFrom(text: string, now: DateTime.Utc): string | undefined { @@ -106,6 +119,12 @@ function resetAtFrom(text: string, now: DateTime.Utc): string | undefined { return delayMs > 0 ? DateTime.formatIso(DateTime.add(now, { milliseconds: delayMs })) : undefined; } +// Reorders STATUS_ERROR's groups to API_ERROR's: HTTP status, message, status name. +function statusError(text: string) { + const match = STATUS_ERROR.exec(text); + return match === null ? null : ([match[0], match[2], match[3], match[1]] as const); +} + function turnFailure( stopReason: EffectAcpSchema.StopReason, step: LastStep | undefined, @@ -114,6 +133,7 @@ function turnFailure( | { readonly failure: OrchestrationV2ProviderFailure; readonly threadDisposition: "reusable" | "broken"; + readonly resumeWith?: string; } | undefined { const errorText = @@ -123,7 +143,7 @@ function turnFailure( // The agent maps only the SDK's QUOTA_EXHAUSTED stop reason to `refusal`. if (errorText === undefined && stopReason !== "refusal") return undefined; const text = errorText ?? ""; - const apiError = API_ERROR.exec(text); + const apiError = API_ERROR.exec(text) ?? statusError(text); const reusable = (failure: OrchestrationV2ProviderFailure) => ({ failure, threadDisposition: "reusable" }) as const; if (stopReason === "refusal" || QUOTA_EXHAUSTED.test(text)) { @@ -159,15 +179,18 @@ function turnFailure( } const [, httpStatus, message, status] = apiError; const rateLimited = httpStatus === "429" || status === "RESOURCE_EXHAUSTED"; - return reusable( - makeProviderFailure({ - class: rateLimited ? "usage_limit" : "provider_error", - message: message!.trim(), - code: status ?? null, - retryable: rateLimited || Number(httpStatus) >= 500, - resetAt: resetAtFrom(text, now) ?? null, - }), - ); + // A 404 before the model answered is an unknown or unentitled model; after, a transient edge error. + const transientNotFound = httpStatus === "404" && step?.reachedModel === true; + const failure = makeProviderFailure({ + class: rateLimited ? "usage_limit" : "provider_error", + message: message!.trim(), + code: status ?? null, + retryable: rateLimited || transientNotFound || Number(httpStatus) >= 500, + resetAt: resetAtFrom(text, now) ?? null, + }); + return transientNotFound + ? { failure, threadDisposition: "reusable", resumeWith: "continue" } + : reusable(failure); } const conversationSegments = ["antigravity-acp", "conversations"]; diff --git a/packages/provider-acp/src/server/adapter.ts b/packages/provider-acp/src/server/adapter.ts index d25274bb9a4c..5a4e798691a5 100644 --- a/packages/provider-acp/src/server/adapter.ts +++ b/packages/provider-acp/src/server/adapter.ts @@ -368,6 +368,8 @@ export function acpPerTurnPromptUsage( interface AcpPromptOutcomeFailure { readonly failure: OrchestrationV2ProviderFailure; readonly threadDisposition: "reusable" | "broken"; + /** The native session can carry on: the turn resends this prompt once before failing. */ + readonly resumeWith?: string; } export interface AcpAdapterV2Flavor { @@ -1670,6 +1672,8 @@ interface ActiveAcpTurn { foregroundIdle: Deferred.Deferred; /** Counts the turn's prompts; only the latest one settles the turn. */ promptSequence: number; + /** The turn already resent a prompt after a resumable outcome failure. */ + outcomeResumed: boolean; /** Clock millis just before `session/prompt` was handed to the runtime. */ promptSentAt: number | null; /** Completed when an update after dispatch matches `cancelAfterPromptStarts.isStart`. */ @@ -8967,7 +8971,7 @@ export const makeAcpAdapterV2 = Effect.fn("makeAcpAdapterV2")(function* ( readonly promptParts: Effect.Success> | null; readonly compactSession: AcpAdapterV2Flavor["compactSession"]; }, - ) => { + ): Effect.Effect => { const sequence = ++context.promptSequence; const superseded = () => context.promptSequence !== sequence; return prompt(superseded).pipe( @@ -9034,6 +9038,43 @@ export const makeAcpAdapterV2 = Effect.fn("makeAcpAdapterV2")(function* ( result.stopReason === "cancelled" ? undefined : yield* settle.checkPromptOutcome(result); + const resumeWith = hiddenFailure?.resumeWith; + if ( + hiddenFailure !== undefined && + resumeWith !== undefined && + steersWithPrompt && + !context.outcomeResumed + ) { + context.outcomeResumed = true; + yield* reportProviderRetry({ + sessionId: context.nativeThreadId, + failure: hiddenFailure.failure, + }); + yield* steerWithPrompt({ + threadId: context.input.threadId, + runId: context.input.runId, + providerThread: context.input.providerThread, + providerTurnId: context.providerTurnId, + message: { ...context.input.message, text: resumeWith, attachments: [] }, + }).pipe( + Effect.catchCause(() => + runRuntimeCallbackAtGeneration( + settle.generation, + Effect.gen(function* () { + if (context.finalized || context.interrupted) return; + yield* Ref.set(runtimeRestartRequired, true); + yield* terminalizeOpenForegroundTools(context, "failed"); + yield* finalizeTurn(context, "failed", hiddenFailure.failure, { + childTurnsContinue: hasSurvivingBackgroundWork(context), + threadDisposition: hiddenFailure.threadDisposition, + }); + }), + ), + ), + Effect.forkIn(sessionScope), + ); + return; + } if (hiddenFailure !== undefined) { yield* Ref.set(runtimeRestartRequired, true); } @@ -9104,7 +9145,12 @@ export const makeAcpAdapterV2 = Effect.fn("makeAcpAdapterV2")(function* ( error.success._tag === "AcpRequestError" && error.success.code === -32800); const failure = Cause.squash(cause); - if (!interrupted && flavor.restartRuntimeAfterPromptError?.(failure) === true) { + // A resumed prompt stands in for an outcome failure, which always restarts. + if ( + !interrupted && + (context.outcomeResumed || + flavor.restartRuntimeAfterPromptError?.(failure) === true) + ) { yield* Ref.set(runtimeRestartRequired, true); } // An agent that answered with an error may still finish tools it started, @@ -9312,6 +9358,7 @@ export const makeAcpAdapterV2 = Effect.fn("makeAcpAdapterV2")(function* ( promptDispatched, foregroundIdle: yield* Deferred.make(), promptSequence: 0, + outcomeResumed: false, promptSentAt: null, promptStarted, promptUsage: undefined,