diff --git a/apps/server/src/orchestration-v2/Adapters/AntigravityAdapterV2.test.ts b/apps/server/src/orchestration-v2/Adapters/AntigravityAdapterV2.test.ts index fca472c6c433e..92f3aca4aeb05 100644 --- a/apps/server/src/orchestration-v2/Adapters/AntigravityAdapterV2.test.ts +++ b/apps/server/src/orchestration-v2/Adapters/AntigravityAdapterV2.test.ts @@ -1,3 +1,5 @@ +import * as NodeSqlite from "node:sqlite"; + import * as NodeCrypto from "@effect/platform-node/NodeCrypto"; import * as NodeServices from "@effect/platform-node/NodeServices"; import { assert, describe, it } from "@effect/vitest"; @@ -2010,6 +2012,138 @@ describe("AntigravityAdapterV2 work after the prompt ends", () => { assert.equal(opened.cancels(), cancels); }).pipe(Effect.provide(layerSession)), ); + + it.effect("fails a deferred-finalize run with resetAt when a wake reply hits quota", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const profileDirectory = yield* fs.makeTempDirectoryScoped(); + const conversations = path.join(profileDirectory, "antigravity-acp", "conversations"); + yield* fs.makeDirectory(conversations, { recursive: true }); + const dbPath = path.join(conversations, "mock-session-1.db"); + const appendStep = (idx: number, type: number, payload: Uint8Array) => { + const database = new NodeSqlite.DatabaseSync(dbPath); + try { + database.exec( + "CREATE TABLE IF NOT EXISTS steps (idx INTEGER PRIMARY KEY, step_type INTEGER, step_payload BLOB)", + ); + database.prepare("INSERT INTO steps VALUES (?, ?, ?)").run(idx, type, payload); + } finally { + database.close(); + } + }; + appendStep(0, 14, Buffer.from("chart the throughput")); + appendStep(1, 15, Buffer.from("running command")); + + const opened = yield* openSessionWithRunningCommand( + { WaitMsBeforeAsync: 10000 }, + undefined, + undefined, + { profileDirectory }, + ); + yield* opened.startTurn(1, false); + yield* opened.throughPromptSettled; + + yield* opened.send({ + sessionUpdate: "tool_call_update", + ...opened.command, + status: "completed", + rawOutput: { output: "Command exited with code 0.", exit_code: 0 }, + }); + appendStep( + 2, + 17, + Buffer.from( + 'RESOURCE_EXHAUSTED (code 429): Individual quota reached. Resets in 123h9m53s.\n{"error":{"code":429,"details":[{"@type":"type.googleapis.com/google.rpc.ErrorInfo","reason":"QUOTA_EXHAUSTED","metadata":{"quotaResetDelay":"123h9m53s","quotaResetTimeStamp":"2026-11-26T11:05:00Z"}}],"status":"RESOURCE_EXHAUSTED"}}', + ), + ); + const debounce = yield* Queue.take(opened.scheduledFinalizes); + yield* TestClock.adjust(debounce); + const terminal = (yield* opened.throughTerminal).at(-1); + if (terminal?.type !== "turn.terminal" || terminal.status !== "failed") { + return assert.fail(`expected failed terminal, got ${JSON.stringify(terminal)}`); + } + assert.deepEqual(terminal.failure, { + class: "usage_limit", + message: "You have reached your Antigravity quota for this period.", + code: "RESOURCE_EXHAUSTED", + retryable: true, + resetAt: "2026-11-26T11:05:00.000Z", + }); + assert.equal(terminal.threadDisposition, "reusable"); + }).pipe(Effect.provide(layerSession)), + ); + + it.effect( + "fails a continuation run with resetAt when a daemon wake hits quota before the run starts", + () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const profileDirectory = yield* fs.makeTempDirectoryScoped(); + const conversations = path.join(profileDirectory, "antigravity-acp", "conversations"); + yield* fs.makeDirectory(conversations, { recursive: true }); + const dbPath = path.join(conversations, "mock-session-1.db"); + const appendStep = (idx: number, type: number, payload: Uint8Array) => { + const database = new NodeSqlite.DatabaseSync(dbPath); + try { + database.exec( + "CREATE TABLE IF NOT EXISTS steps (idx INTEGER PRIMARY KEY, step_type INTEGER, step_payload BLOB)", + ); + database.prepare("INSERT INTO steps VALUES (?, ?, ?)").run(idx, type, payload); + } finally { + database.close(); + } + }; + appendStep(0, 14, Buffer.from("start daemon")); + appendStep(1, 15, Buffer.from("daemon started")); + + const opened = yield* openSessionWithRunningCommand( + { IsDaemon: true, WaitMsBeforeAsync: 500 }, + undefined, + undefined, + { profileDirectory }, + ); + yield* opened.startTurn(1, false); + const promptTerminal = (yield* opened.throughTerminal).at(-1); + assert.equal( + promptTerminal?.type === "turn.terminal" && promptTerminal.status, + "completed", + ); + + // Daemon exits and the agent's wake reply hits quota before OrchestrationV2 starts turn 2. + yield* opened.send({ + sessionUpdate: "tool_call_update", + ...opened.command, + status: "completed", + rawOutput: { output: "Command exited with code 0.", exit_code: 0 }, + }); + appendStep( + 2, + 17, + Buffer.from( + 'RESOURCE_EXHAUSTED (code 429): Individual quota reached. Resets in 11m36s.\n{"error":{"code":429,"details":[{"@type":"type.googleapis.com/google.rpc.ErrorInfo","reason":"QUOTA_EXHAUSTED","metadata":{"quotaResetDelay":"11m36.055910958s","quotaResetTimeStamp":"2026-11-26T11:05:00Z"}}],"status":"RESOURCE_EXHAUSTED"}}', + ), + ); + assert.lengthOf(opened.offers, 1); + yield* opened.offers[0]!.dispatchIfCurrent!(Effect.void); + yield* opened.startTurn(2, true); + const debounce = yield* Queue.take(opened.scheduledFinalizes); + yield* TestClock.adjust(debounce); + const terminal = (yield* opened.throughTerminal).at(-1); + if (terminal?.type !== "turn.terminal" || terminal.status !== "failed") { + return assert.fail(`expected failed terminal, got ${JSON.stringify(terminal)}`); + } + assert.deepEqual(terminal.failure, { + class: "usage_limit", + message: "You have reached your Antigravity quota for this period.", + code: "RESOURCE_EXHAUSTED", + retryable: true, + resetAt: "2026-11-26T11:05:00.000Z", + }); + assert.equal(terminal.threadDisposition, "reusable"); + }).pipe(Effect.provide(layerSession)), + ); }); describe("AntigravityAdapterV2 subagent reads", () => { diff --git a/apps/server/src/provider/acp/AntigravityTurnOutcome.test.ts b/apps/server/src/provider/acp/AntigravityTurnOutcome.test.ts index b6094e638285d..e6101193e8534 100644 --- a/apps/server/src/provider/acp/AntigravityTurnOutcome.test.ts +++ b/apps/server/src/provider/acp/AntigravityTurnOutcome.test.ts @@ -50,6 +50,31 @@ const STATUS_NOT_FOUND_ERROR = Buffer.concat([ 'NOT_FOUND (code 404): Requested entity was not found.\u001a\u0080\u0001HTTP 404 Not Found\n\n{"error":{"code":404,"message":"Requested entity was not found.","status":"NOT_FOUND"}}', ), ]); +const STATUS_QUOTA_ERROR = Buffer.concat([ + Buffer.from([0x12, 0x68]), + Buffer.from( + 'RESOURCE_EXHAUSTED (code 429): Individual quota reached. Please upgrade your subscription to increase your limits. Resets in 11m36s.\u001a\u0080\u0005HTTP 429 Too Many Requests\n\n{\n "error": {\n "code": 429,\n "details": [\n {\n "@type": "type.googleapis.com/google.rpc.ErrorInfo",\n "domain": "cloudcode-pa.googleapis.com",\n "metadata": {\n "model": "gemini-4-neon-high",\n "quotaResetDelay": "11m36.055910958s",\n "quotaResetTimeStamp": "2026-11-21T04:36:37Z"\n },\n "reason": "QUOTA_EXHAUSTED"\n },\n {\n "@type": "type.googleapis.com/google.rpc.RetryInfo",\n "retryDelay": "696.055910958s"\n }\n ],\n "message": "Individual quota reached. Please upgrade your subscription to increase your limits. Resets in 11m36s.",\n "status": "RESOURCE_EXHAUSTED"\n }\n}', + ), +]); +const STATUS_QUOTA_DELAY_ERROR = Buffer.concat([ + Buffer.from([0x12, 0x68]), + Buffer.from( + 'RESOURCE_EXHAUSTED (code 429): Individual quota reached.\u001a\u0080\u0002HTTP 429 Too Many Requests\n\n{"error":{"code":429,"details":[{"@type":"type.googleapis.com/google.rpc.RetryInfo","retryDelay":"42s"},{"@type":"type.googleapis.com/google.rpc.ErrorInfo","reason":"QUOTA_EXHAUSTED","metadata":{"quotaResetDelay":"11m36.055910958s","quotaResetTimeStamp":"1970-01-01T00:00:00Z"}}],"status":"RESOURCE_EXHAUSTED"}}', + ), +]); +const STATUS_QUOTA_ZERO_DELAY_ERROR = Buffer.concat([ + Buffer.from([0x12, 0x68]), + Buffer.from( + 'RESOURCE_EXHAUSTED (code 429): Individual quota reached.\u001a\u0080\u0002HTTP 429 Too Many Requests\n\n{"error":{"code":429,"details":[{"@type":"type.googleapis.com/google.rpc.ErrorInfo","reason":"QUOTA_EXHAUSTED","metadata":{"quotaResetDelay":"0s","quotaResetTimeStamp":"1970-01-01T00:00:00Z"}},{"@type":"type.googleapis.com/google.rpc.RetryInfo","retryDelay":"42s"}],"status":"RESOURCE_EXHAUSTED"}}', + ), +]); +const STATUS_RATE_LIMIT_ERROR = Buffer.concat([ + // 0x4e (78 = 'N') is the protobuf length varint of the status string and must not corrupt status. + Buffer.from([0x12, 0x4e]), + Buffer.from( + 'RESOURCE_EXHAUSTED (code 429): Resource has been exhausted (e.g. check quota).\u001a\u0080\u0002HTTP 429 Too Many Requests\n\n{"error":{"code":429,"details":[{"@type":"type.googleapis.com/google.rpc.ErrorInfo","reason":"RATE_LIMIT_EXCEEDED"},{"@type":"type.googleapis.com/google.rpc.RetryInfo","retryDelay":"42s"}],"status":"RESOURCE_EXHAUSTED"}}', + ), +]); // The model's reply quoting the agent's quota notice is an ordinary planner response. const QUOTED_NOTICE = Buffer.from( "Usage Limit Reached\n\nYou have reached your current quota for this period. Your limit will reset in 1 hour.", @@ -137,6 +162,58 @@ describe("watchAntigravityPromptOutcome", () => { resetAt: "1970-01-01T00:00:42.000Z", }), ); + assert.deepEqual( + yield* turn([ + [USER_INPUT, input], + [ERROR_MESSAGE, STATUS_QUOTA_ERROR], + ]), + reusable({ + class: "usage_limit", + message: "You have reached your Antigravity quota for this period.", + code: "RESOURCE_EXHAUSTED", + retryable: true, + resetAt: "2026-11-21T04:36:37.000Z", + }), + ); + assert.deepEqual( + yield* turn([ + [USER_INPUT, input], + [ERROR_MESSAGE, STATUS_QUOTA_DELAY_ERROR], + ]), + reusable({ + class: "usage_limit", + message: "You have reached your Antigravity quota for this period.", + code: "RESOURCE_EXHAUSTED", + retryable: true, + resetAt: "1970-01-01T00:11:36.056Z", + }), + ); + assert.deepEqual( + yield* turn([ + [USER_INPUT, input], + [ERROR_MESSAGE, STATUS_QUOTA_ZERO_DELAY_ERROR], + ]), + reusable({ + class: "usage_limit", + message: "You have reached your Antigravity quota for this period.", + code: "RESOURCE_EXHAUSTED", + retryable: true, + resetAt: "1970-01-01T00:00:42.000Z", + }), + ); + assert.deepEqual( + yield* turn([ + [USER_INPUT, input], + [ERROR_MESSAGE, STATUS_RATE_LIMIT_ERROR], + ]), + reusable({ + class: "usage_limit", + message: "Resource has been exhausted (e.g. check quota).", + code: "RESOURCE_EXHAUSTED", + retryable: true, + resetAt: "1970-01-01T00:00:42.000Z", + }), + ); assert.deepEqual( yield* turn([ [USER_INPUT, input], diff --git a/apps/server/src/provider/acp/AntigravityTurnOutcome.ts b/apps/server/src/provider/acp/AntigravityTurnOutcome.ts index 32eee40f008ae..40b8ec34a2f2f 100644 --- a/apps/server/src/provider/acp/AntigravityTurnOutcome.ts +++ b/apps/server/src/provider/acp/AntigravityTurnOutcome.ts @@ -21,17 +21,25 @@ const PLANNER_RESPONSE_STEP_TYPE = 15; const MAX_ERROR_PAYLOAD_BYTES = 16_384; // The Go harness's rendering of a Google API error, as the agent's api_errors.py reads it. const API_ERROR = /Error (\d{3}), Message: (.+?), Status: ([A-Z_]+)/s; -// Antigravity 1.5 with a Google account writes "STATUS (code NNN): message". -const STATUS_ERROR = /(? + new RegExp( + `(?:"${field}"\\s*:\\s*"(${GO_DURATION})"|\\b${field}:(${GO_DURATION})(?=[\\s\\],}]|$))`, + ); +const RESET_TIMESTAMP = + /(?:"quotaResetTimeStamp"\s*:\s*"([^"\s]+)"|\bquotaResetTimeStamp:([^\s\],}]+))/; +const RESET_DELAY = delayPattern("quotaResetDelay"); // google.rpc.RetryInfo, which plain rate limits carry instead of a quota reset. -const RETRY_DELAY = /\bretryDelay:((?:\d+(?:\.\d+)?(?:ns|us|µs|μs|ms|h|m|s))+)(?=[\s\]])/; +const RETRY_DELAY = delayPattern("retryDelay"); const GO_DURATION_PART = /(\d+(?:\.\d+)?)(ns|us|µs|μs|ms|h|m|s)/g; const GO_DURATION_UNIT_MS: Record = { ns: 1e-6, @@ -106,17 +114,34 @@ const lastStepAfter = }; }; -function resetAtFrom(text: string, now: DateTime.Utc): string | undefined { - // makeProviderFailure drops a timestamp that does not parse, so fall back to the delay. - const timestamp = RESET_TIMESTAMP.exec(text)?.[1]; - if (timestamp !== undefined && Option.isSome(DateTime.make(timestamp))) return timestamp; - const delay = RESET_DELAY.exec(text)?.[1] ?? RETRY_DELAY.exec(text)?.[1]; +function matchedValue(match: RegExpExecArray | null): string | undefined { + return match?.[1] ?? match?.[2]; +} + +function parsePositiveDelayMs(delay: string | undefined): number | undefined { if (delay === undefined) return undefined; let delayMs = 0; for (const [, amount, unit] of delay.matchAll(GO_DURATION_PART)) { delayMs += Number(amount) * (GO_DURATION_UNIT_MS[unit!] ?? 0); } - return delayMs > 0 ? DateTime.formatIso(DateTime.add(now, { milliseconds: delayMs })) : undefined; + return delayMs > 0 ? Math.ceil(delayMs) : undefined; +} + +function resetAtFrom(text: string, now: DateTime.Utc): string | undefined { + // makeProviderFailure drops an unparsable timestamp, and recovery ignores one at or before now. + const timestamp = matchedValue(RESET_TIMESTAMP.exec(text)); + const parsedTimestamp = timestamp === undefined ? Option.none() : DateTime.make(timestamp); + if (Option.isSome(parsedTimestamp) && DateTime.isGreaterThan(parsedTimestamp.value, now)) { + return timestamp; + } + const delayMs = + parsePositiveDelayMs(matchedValue(RESET_DELAY.exec(text))) ?? + parsePositiveDelayMs(matchedValue(RETRY_DELAY.exec(text))); + return delayMs !== undefined + ? DateTime.formatIso(DateTime.add(now, { milliseconds: delayMs })) + : Option.isSome(parsedTimestamp) + ? timestamp + : undefined; } // Reorders STATUS_ERROR's groups to API_ERROR's: HTTP status, message, status name. diff --git a/packages/provider-acp/src/server/adapter.ts b/packages/provider-acp/src/server/adapter.ts index cf7bb6c0f92fd..7acbccaf70d94 100644 --- a/packages/provider-acp/src/server/adapter.ts +++ b/packages/provider-acp/src/server/adapter.ts @@ -1721,6 +1721,10 @@ interface ActiveAcpTurn { promptSequence: number; /** The turn already resent a prompt after a resumable outcome failure. */ outcomeResumed: boolean; + /** Outcome check armed for the turn's latest prompt or wake, reused if finalization defers. */ + checkPromptOutcome: + | Effect.Success>> + | undefined; /** Clock millis just before `session/prompt` was handed to the runtime. */ promptSentAt: number | null; /** Completed when an update after dispatch matches `cancelAfterPromptStarts.isStart`. */ @@ -2340,6 +2344,12 @@ export const makeAcpAdapterV2 = Effect.fn("makeAcpAdapterV2")(function* ( flavor.compactSession !== undefined || flavor.compactViaAdvertisedCommand === true; const runtimeRestartRequired = yield* Ref.make(false); + const pendingContinuationOutcome = yield* Ref.make<{ + readonly sessionId: string; + readonly check: Effect.Success< + ReturnType> + >; + } | null>(null); const runtimeTeardownState = yield* Ref.make({ _tag: "Idle" }); const runtimeCallbackGeneration = yield* Ref.make(0); const runtimeCallbackGenerationCounter = yield* Ref.make(0); @@ -7933,6 +7943,7 @@ export const makeAcpAdapterV2 = Effect.fn("makeAcpAdapterV2")(function* ( ) { rememberTerminalEnvironment(candidate.sessionId, yield* readMcpContext(threadId)); yield* Ref.set(runtimeRestartRequired, false); + yield* Ref.set(pendingContinuationOutcome, null); yield* Ref.set(activeSessionId, candidate.sessionId); yield* setActiveSessionSetup(candidate); yield* Ref.set(activeSelection, null); @@ -7985,6 +7996,7 @@ export const makeAcpAdapterV2 = Effect.fn("makeAcpAdapterV2")(function* ( }); itemIdentityVersion = 2; yield* Ref.set(runtimeRestartRequired, false); + yield* Ref.set(pendingContinuationOutcome, null); prepareClaimableTerminalEnvironment(yield* readMcpContext(input.threadId)); return yield* startAcpRuntime(input.threadId); }); @@ -8706,6 +8718,18 @@ export const makeAcpAdapterV2 = Effect.fn("makeAcpAdapterV2")(function* ( if (settledStatus === "completed" && !directStopQuarantine) { yield* detachBackgroundCommands(context); } + yield* Ref.set( + pendingContinuationOutcome, + postSettleContinuationEnabled && + settledStatus === "completed" && + !directStopQuarantine && + context.checkPromptOutcome !== undefined + ? { + sessionId: context.nativeThreadId, + check: context.checkPromptOutcome, + } + : null, + ); steerInjections.clear(); if ( postSettleContinuationEnabled && @@ -8733,6 +8757,7 @@ export const makeAcpAdapterV2 = Effect.fn("makeAcpAdapterV2")(function* ( if (hasDeferredBackgroundWork(context)) return; context.backgroundFinalizeGeneration += 1; const generation = context.backgroundFinalizeGeneration; + const callbackGeneration = yield* Ref.get(runtimeCallbackGeneration); const replyOpen = context.replyOpen; const debounce = replyOpen ? ACP_OPEN_REPLY_QUIET_LIMIT : ACP_DEFERRED_FINALIZE_DEBOUNCE; yield* Effect.gen(function* () { @@ -8753,7 +8778,28 @@ export const makeAcpAdapterV2 = Effect.fn("makeAcpAdapterV2")(function* ( yield* scheduleDeferredFinalize(context); return; } + const isCurrent = Effect.gen(function* () { + if (!stillCurrent()) return false; + if ((yield* Ref.get(runtimeTeardownState))._tag !== "Idle") return false; + return (yield* Ref.get(runtimeCallbackGeneration)) === callbackGeneration; + }); + if (!(yield* isCurrent)) return; const status = context.promptSettledStatus ?? "completed"; + const hiddenFailure = + status === "completed" && context.checkPromptOutcome !== undefined + ? yield* context.checkPromptOutcome({ stopReason: "end_turn" }) + : undefined; + if (!(yield* isCurrent)) return; + if (hiddenFailure !== undefined) { + yield* terminalizeOpenForegroundTools(context, "failed"); + if (!(yield* isCurrent)) return; + yield* Ref.set(runtimeRestartRequired, true); + yield* finalizeTurn(context, "failed", hiddenFailure.failure, { + childTurnsContinue: hasSurvivingBackgroundWork(context), + threadDisposition: hiddenFailure.threadDisposition, + }); + return; + } yield* finalizeTurn(context, status); }).pipe(Effect.forkIn(sessionScope), Effect.asVoid); yield* options.testHooks?.onDeferredFinalizeScheduled?.(debounce) ?? Effect.void; @@ -9005,6 +9051,7 @@ export const makeAcpAdapterV2 = Effect.fn("makeAcpAdapterV2")(function* ( if (!restartRequired) return false; yield* restartAcpRuntime(threadId); yield* Ref.set(runtimeRestartRequired, false); + yield* Ref.set(pendingContinuationOutcome, null); yield* Ref.set(activeSessionId, null); yield* Ref.set(activeSessionSetup, null); yield* Ref.set(activeSelection, null); @@ -9442,6 +9489,7 @@ export const makeAcpAdapterV2 = Effect.fn("makeAcpAdapterV2")(function* ( foregroundIdle: yield* Deferred.make(), promptSequence: 0, outcomeResumed: false, + checkPromptOutcome: undefined, promptSentAt: null, promptStarted, promptUsage: undefined, @@ -9525,7 +9573,14 @@ export const makeAcpAdapterV2 = Effect.fn("makeAcpAdapterV2")(function* ( // start before that continuation and must leave its buffer intact. yield* projectDeferredCarryoverTerminals(rehydratedCarryoverSubagents); // Retain projected terminals during wake drain so monotonic guards reject replayed spawns. + const continuationOutcome = yield* Ref.getAndSet(pendingContinuationOutcome, null); if (isContinuationTurn) { + context.checkPromptOutcome = + continuationOutcome?.sessionId === requestedSessionId + ? continuationOutcome.check + : flavor.watchPromptOutcome === undefined + ? undefined + : yield* flavor.watchPromptOutcome(requestedSessionId); yield* Ref.set(continuationRequested, false); yield* Ref.update(wakeReports, ({ reports, offered }) => ({ reports: new Map([...reports].filter(([key]) => !offered.has(key))), @@ -9601,6 +9656,7 @@ export const makeAcpAdapterV2 = Effect.fn("makeAcpAdapterV2")(function* ( flavor.watchPromptOutcome === undefined ? undefined : yield* flavor.watchPromptOutcome(requestedSessionId); + context.checkPromptOutcome = checkPromptOutcome; const promptGeneration = yield* Ref.get(runtimeCallbackGeneration); const promptOptions = { dispatched: context.promptDispatched, @@ -9701,6 +9757,7 @@ export const makeAcpAdapterV2 = Effect.fn("makeAcpAdapterV2")(function* ( } context.promptSettled = false; context.promptSettledStatus = null; + context.checkPromptOutcome = checkPromptOutcome; context.backgroundFinalizeGeneration += 1; context.promptSentAt = yield* Clock.currentTimeMillis; context.lastProgressAt = context.promptSentAt;