Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
134 changes: 134 additions & 0 deletions apps/server/src/orchestration-v2/Adapters/AntigravityAdapterV2.test.ts
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
import * as NodeSqlite from "node:sqlite";

import * as NodeCrypto from "@effect/platform-node/NodeCrypto";
import * as NodeServices from "@effect/platform-node/NodeServices";
import { assert, describe, it } from "@effect/vitest";
Expand Down Expand Up @@ -2010,6 +2012,138 @@ describe("AntigravityAdapterV2 work after the prompt ends", () => {
assert.equal(opened.cancels(), cancels);
}).pipe(Effect.provide(layerSession)),
);

it.effect("fails a deferred-finalize run with resetAt when a wake reply hits quota", () =>
Effect.gen(function* () {
const fs = yield* FileSystem.FileSystem;
const path = yield* Path.Path;
const profileDirectory = yield* fs.makeTempDirectoryScoped();
const conversations = path.join(profileDirectory, "antigravity-acp", "conversations");
yield* fs.makeDirectory(conversations, { recursive: true });
const dbPath = path.join(conversations, "mock-session-1.db");
const appendStep = (idx: number, type: number, payload: Uint8Array) => {
const database = new NodeSqlite.DatabaseSync(dbPath);
try {
database.exec(
"CREATE TABLE IF NOT EXISTS steps (idx INTEGER PRIMARY KEY, step_type INTEGER, step_payload BLOB)",
);
database.prepare("INSERT INTO steps VALUES (?, ?, ?)").run(idx, type, payload);
} finally {
database.close();
}
};
appendStep(0, 14, Buffer.from("chart the throughput"));
appendStep(1, 15, Buffer.from("running command"));

const opened = yield* openSessionWithRunningCommand(
{ WaitMsBeforeAsync: 10000 },
undefined,
undefined,
{ profileDirectory },
);
yield* opened.startTurn(1, false);
yield* opened.throughPromptSettled;

yield* opened.send({
sessionUpdate: "tool_call_update",
...opened.command,
status: "completed",
rawOutput: { output: "Command exited with code 0.", exit_code: 0 },
});
appendStep(
2,
17,
Buffer.from(
'RESOURCE_EXHAUSTED (code 429): Individual quota reached. Resets in 123h9m53s.\n{"error":{"code":429,"details":[{"@type":"type.googleapis.com/google.rpc.ErrorInfo","reason":"QUOTA_EXHAUSTED","metadata":{"quotaResetDelay":"123h9m53s","quotaResetTimeStamp":"2026-11-26T11:05:00Z"}}],"status":"RESOURCE_EXHAUSTED"}}',
),
);
const debounce = yield* Queue.take(opened.scheduledFinalizes);
yield* TestClock.adjust(debounce);
const terminal = (yield* opened.throughTerminal).at(-1);
if (terminal?.type !== "turn.terminal" || terminal.status !== "failed") {
return assert.fail(`expected failed terminal, got ${JSON.stringify(terminal)}`);
}
assert.deepEqual(terminal.failure, {
class: "usage_limit",
message: "You have reached your Antigravity quota for this period.",
code: "RESOURCE_EXHAUSTED",
retryable: true,
resetAt: "2026-11-26T11:05:00.000Z",
});
assert.equal(terminal.threadDisposition, "reusable");
}).pipe(Effect.provide(layerSession)),
);

it.effect(
"fails a continuation run with resetAt when a daemon wake hits quota before the run starts",
() =>
Effect.gen(function* () {
const fs = yield* FileSystem.FileSystem;
const path = yield* Path.Path;
const profileDirectory = yield* fs.makeTempDirectoryScoped();
const conversations = path.join(profileDirectory, "antigravity-acp", "conversations");
yield* fs.makeDirectory(conversations, { recursive: true });
const dbPath = path.join(conversations, "mock-session-1.db");
const appendStep = (idx: number, type: number, payload: Uint8Array) => {
const database = new NodeSqlite.DatabaseSync(dbPath);
try {
database.exec(
"CREATE TABLE IF NOT EXISTS steps (idx INTEGER PRIMARY KEY, step_type INTEGER, step_payload BLOB)",
);
database.prepare("INSERT INTO steps VALUES (?, ?, ?)").run(idx, type, payload);
} finally {
database.close();
}
};
appendStep(0, 14, Buffer.from("start daemon"));
appendStep(1, 15, Buffer.from("daemon started"));

const opened = yield* openSessionWithRunningCommand(
{ IsDaemon: true, WaitMsBeforeAsync: 500 },
undefined,
undefined,
{ profileDirectory },
);
yield* opened.startTurn(1, false);
const promptTerminal = (yield* opened.throughTerminal).at(-1);
assert.equal(
promptTerminal?.type === "turn.terminal" && promptTerminal.status,
"completed",
);

// Daemon exits and the agent's wake reply hits quota before OrchestrationV2 starts turn 2.
yield* opened.send({
sessionUpdate: "tool_call_update",
...opened.command,
status: "completed",
rawOutput: { output: "Command exited with code 0.", exit_code: 0 },
});
appendStep(
2,
17,
Buffer.from(
'RESOURCE_EXHAUSTED (code 429): Individual quota reached. Resets in 11m36s.\n{"error":{"code":429,"details":[{"@type":"type.googleapis.com/google.rpc.ErrorInfo","reason":"QUOTA_EXHAUSTED","metadata":{"quotaResetDelay":"11m36.055910958s","quotaResetTimeStamp":"2026-11-26T11:05:00Z"}}],"status":"RESOURCE_EXHAUSTED"}}',
),
);
assert.lengthOf(opened.offers, 1);
yield* opened.offers[0]!.dispatchIfCurrent!(Effect.void);
yield* opened.startTurn(2, true);
const debounce = yield* Queue.take(opened.scheduledFinalizes);
yield* TestClock.adjust(debounce);
const terminal = (yield* opened.throughTerminal).at(-1);
if (terminal?.type !== "turn.terminal" || terminal.status !== "failed") {
return assert.fail(`expected failed terminal, got ${JSON.stringify(terminal)}`);
}
assert.deepEqual(terminal.failure, {
class: "usage_limit",
message: "You have reached your Antigravity quota for this period.",
code: "RESOURCE_EXHAUSTED",
retryable: true,
resetAt: "2026-11-26T11:05:00.000Z",
});
assert.equal(terminal.threadDisposition, "reusable");
}).pipe(Effect.provide(layerSession)),
);
});

describe("AntigravityAdapterV2 subagent reads", () => {
Expand Down
77 changes: 77 additions & 0 deletions apps/server/src/provider/acp/AntigravityTurnOutcome.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,31 @@ const STATUS_NOT_FOUND_ERROR = Buffer.concat([
'NOT_FOUND (code 404): Requested entity was not found.\u001a\u0080\u0001HTTP 404 Not Found\n\n{"error":{"code":404,"message":"Requested entity was not found.","status":"NOT_FOUND"}}',
),
]);
const STATUS_QUOTA_ERROR = Buffer.concat([
Buffer.from([0x12, 0x68]),
Buffer.from(
'RESOURCE_EXHAUSTED (code 429): Individual quota reached. Please upgrade your subscription to increase your limits. Resets in 11m36s.\u001a\u0080\u0005HTTP 429 Too Many Requests\n\n{\n "error": {\n "code": 429,\n "details": [\n {\n "@type": "type.googleapis.com/google.rpc.ErrorInfo",\n "domain": "cloudcode-pa.googleapis.com",\n "metadata": {\n "model": "gemini-4-neon-high",\n "quotaResetDelay": "11m36.055910958s",\n "quotaResetTimeStamp": "2026-11-21T04:36:37Z"\n },\n "reason": "QUOTA_EXHAUSTED"\n },\n {\n "@type": "type.googleapis.com/google.rpc.RetryInfo",\n "retryDelay": "696.055910958s"\n }\n ],\n "message": "Individual quota reached. Please upgrade your subscription to increase your limits. Resets in 11m36s.",\n "status": "RESOURCE_EXHAUSTED"\n }\n}',
),
]);
const STATUS_QUOTA_DELAY_ERROR = Buffer.concat([
Buffer.from([0x12, 0x68]),
Buffer.from(
'RESOURCE_EXHAUSTED (code 429): Individual quota reached.\u001a\u0080\u0002HTTP 429 Too Many Requests\n\n{"error":{"code":429,"details":[{"@type":"type.googleapis.com/google.rpc.RetryInfo","retryDelay":"42s"},{"@type":"type.googleapis.com/google.rpc.ErrorInfo","reason":"QUOTA_EXHAUSTED","metadata":{"quotaResetDelay":"11m36.055910958s","quotaResetTimeStamp":"1970-01-01T00:00:00Z"}}],"status":"RESOURCE_EXHAUSTED"}}',
),
]);
const STATUS_QUOTA_ZERO_DELAY_ERROR = Buffer.concat([
Buffer.from([0x12, 0x68]),
Buffer.from(
'RESOURCE_EXHAUSTED (code 429): Individual quota reached.\u001a\u0080\u0002HTTP 429 Too Many Requests\n\n{"error":{"code":429,"details":[{"@type":"type.googleapis.com/google.rpc.ErrorInfo","reason":"QUOTA_EXHAUSTED","metadata":{"quotaResetDelay":"0s","quotaResetTimeStamp":"1970-01-01T00:00:00Z"}},{"@type":"type.googleapis.com/google.rpc.RetryInfo","retryDelay":"42s"}],"status":"RESOURCE_EXHAUSTED"}}',
),
]);
const STATUS_RATE_LIMIT_ERROR = Buffer.concat([
// 0x4e (78 = 'N') is the protobuf length varint of the status string and must not corrupt status.
Buffer.from([0x12, 0x4e]),
Buffer.from(
'RESOURCE_EXHAUSTED (code 429): Resource has been exhausted (e.g. check quota).\u001a\u0080\u0002HTTP 429 Too Many Requests\n\n{"error":{"code":429,"details":[{"@type":"type.googleapis.com/google.rpc.ErrorInfo","reason":"RATE_LIMIT_EXCEEDED"},{"@type":"type.googleapis.com/google.rpc.RetryInfo","retryDelay":"42s"}],"status":"RESOURCE_EXHAUSTED"}}',
),
]);
// The model's reply quoting the agent's quota notice is an ordinary planner response.
const QUOTED_NOTICE = Buffer.from(
"Usage Limit Reached\n\nYou have reached your current quota for this period. Your limit will reset in 1 hour.",
Expand Down Expand Up @@ -137,6 +162,58 @@ describe("watchAntigravityPromptOutcome", () => {
resetAt: "1970-01-01T00:00:42.000Z",
}),
);
assert.deepEqual(
yield* turn([
[USER_INPUT, input],
[ERROR_MESSAGE, STATUS_QUOTA_ERROR],
]),
reusable({
class: "usage_limit",
message: "You have reached your Antigravity quota for this period.",
code: "RESOURCE_EXHAUSTED",
retryable: true,
resetAt: "2026-11-21T04:36:37.000Z",
}),
);
assert.deepEqual(
yield* turn([
[USER_INPUT, input],
[ERROR_MESSAGE, STATUS_QUOTA_DELAY_ERROR],
]),
reusable({
class: "usage_limit",
message: "You have reached your Antigravity quota for this period.",
code: "RESOURCE_EXHAUSTED",
retryable: true,
resetAt: "1970-01-01T00:11:36.056Z",
}),
);
assert.deepEqual(
yield* turn([
[USER_INPUT, input],
[ERROR_MESSAGE, STATUS_QUOTA_ZERO_DELAY_ERROR],
]),
reusable({
class: "usage_limit",
message: "You have reached your Antigravity quota for this period.",
code: "RESOURCE_EXHAUSTED",
retryable: true,
resetAt: "1970-01-01T00:00:42.000Z",
}),
);
assert.deepEqual(
yield* turn([
[USER_INPUT, input],
[ERROR_MESSAGE, STATUS_RATE_LIMIT_ERROR],
]),
reusable({
class: "usage_limit",
message: "Resource has been exhausted (e.g. check quota).",
code: "RESOURCE_EXHAUSTED",
retryable: true,
resetAt: "1970-01-01T00:00:42.000Z",
}),
);
assert.deepEqual(
yield* turn([
[USER_INPUT, input],
Expand Down
49 changes: 37 additions & 12 deletions apps/server/src/provider/acp/AntigravityTurnOutcome.ts
Original file line number Diff line number Diff line change
Expand Up @@ -21,17 +21,25 @@ const PLANNER_RESPONSE_STEP_TYPE = 15;
const MAX_ERROR_PAYLOAD_BYTES = 16_384;
// The Go harness's rendering of a Google API error, as the agent's api_errors.py reads it.
const API_ERROR = /Error (\d{3}), Message: (.+?), Status: ([A-Z_]+)/s;
// Antigravity 1.5 with a Google account writes "STATUS (code NNN): message".
const STATUS_ERROR = /(?<![A-Z_])([A-Z_]+) \(code (\d{3})\): (\P{Cc}+)/u;
const QUOTA_EXHAUSTED = /\breason:QUOTA_EXHAUSTED\b/;
// Antigravity 1.5 with a Google account writes "STATUS (code NNN): message" in a protobuf string
// whose length varint (65-90, 95) can be an uppercase ASCII byte immediately before STATUS.
const STATUS_ERROR =
/(CANCELLED|UNKNOWN|INVALID_ARGUMENT|DEADLINE_EXCEEDED|NOT_FOUND|ALREADY_EXISTS|PERMISSION_DENIED|RESOURCE_EXHAUSTED|FAILED_PRECONDITION|ABORTED|OUT_OF_RANGE|UNIMPLEMENTED|INTERNAL|UNAVAILABLE|DATA_LOSS|UNAUTHENTICATED) \(code (\d{3})\): (\P{Cc}+)/u;
const QUOTA_EXHAUSTED = /(?:"reason"\s*:\s*"QUOTA_EXHAUSTED"|\breason:QUOTA_EXHAUSTED\b)/;
// Claude rejects a saved thinking block without a valid signature. It sits in the native
// history, so every later prompt to that session fails the same way.
const INVALID_THINKING_SIGNATURE =
/\bmessages\.\d+\.content\.\d+(?:\.thinking\.signature: Field required|: Invalid `signature` in `thinking` block)/;
const RESET_TIMESTAMP = /\bquotaResetTimeStamp:([^\s\]]+)/;
const RESET_DELAY = /\bquotaResetDelay:((?:\d+(?:\.\d+)?(?:ns|us|µs|μs|ms|h|m|s))+)(?=[\s\]])/;
const GO_DURATION = "(?:\\d+(?:\\.\\d+)?(?:ns|us|µs|μs|ms|h|m|s))+";
const delayPattern = (field: string) =>
new RegExp(
`(?:"${field}"\\s*:\\s*"(${GO_DURATION})"|\\b${field}:(${GO_DURATION})(?=[\\s\\],}]|$))`,
);
const RESET_TIMESTAMP =
/(?:"quotaResetTimeStamp"\s*:\s*"([^"\s]+)"|\bquotaResetTimeStamp:([^\s\],}]+))/;
const RESET_DELAY = delayPattern("quotaResetDelay");
// google.rpc.RetryInfo, which plain rate limits carry instead of a quota reset.
const RETRY_DELAY = /\bretryDelay:((?:\d+(?:\.\d+)?(?:ns|us|µs|μs|ms|h|m|s))+)(?=[\s\]])/;
const RETRY_DELAY = delayPattern("retryDelay");
const GO_DURATION_PART = /(\d+(?:\.\d+)?)(ns|us|µs|μs|ms|h|m|s)/g;
const GO_DURATION_UNIT_MS: Record<string, number> = {
ns: 1e-6,
Expand Down Expand Up @@ -106,17 +114,34 @@ const lastStepAfter =
};
};

function resetAtFrom(text: string, now: DateTime.Utc): string | undefined {
// makeProviderFailure drops a timestamp that does not parse, so fall back to the delay.
const timestamp = RESET_TIMESTAMP.exec(text)?.[1];
if (timestamp !== undefined && Option.isSome(DateTime.make(timestamp))) return timestamp;
const delay = RESET_DELAY.exec(text)?.[1] ?? RETRY_DELAY.exec(text)?.[1];
function matchedValue(match: RegExpExecArray | null): string | undefined {
return match?.[1] ?? match?.[2];
}

function parsePositiveDelayMs(delay: string | undefined): number | undefined {
if (delay === undefined) return undefined;
let delayMs = 0;
for (const [, amount, unit] of delay.matchAll(GO_DURATION_PART)) {
delayMs += Number(amount) * (GO_DURATION_UNIT_MS[unit!] ?? 0);
}
return delayMs > 0 ? DateTime.formatIso(DateTime.add(now, { milliseconds: delayMs })) : undefined;
return delayMs > 0 ? Math.ceil(delayMs) : undefined;
}

function resetAtFrom(text: string, now: DateTime.Utc): string | undefined {
// makeProviderFailure drops an unparsable timestamp, and recovery ignores one at or before now.
const timestamp = matchedValue(RESET_TIMESTAMP.exec(text));
const parsedTimestamp = timestamp === undefined ? Option.none() : DateTime.make(timestamp);
if (Option.isSome(parsedTimestamp) && DateTime.isGreaterThan(parsedTimestamp.value, now)) {
return timestamp;
}
const delayMs =
parsePositiveDelayMs(matchedValue(RESET_DELAY.exec(text))) ??
parsePositiveDelayMs(matchedValue(RETRY_DELAY.exec(text)));
return delayMs !== undefined
? DateTime.formatIso(DateTime.add(now, { milliseconds: delayMs }))
: Option.isSome(parsedTimestamp)
? timestamp
: undefined;
}

// Reorders STATUS_ERROR's groups to API_ERROR's: HTTP status, message, status name.
Expand Down
Loading
Loading