diff --git a/apps/server/src/provider/Drivers/antigravityUsageReader.test.ts b/apps/server/src/provider/Drivers/antigravityUsageReader.test.ts index 8fd031d43442e..04ddb5ada3787 100644 --- a/apps/server/src/provider/Drivers/antigravityUsageReader.test.ts +++ b/apps/server/src/provider/Drivers/antigravityUsageReader.test.ts @@ -62,6 +62,10 @@ function protoText(field: number, value: string): number[] { return protoBytes(field, [...Buffer.from(value)]); } +function lastStepIndex(index: number): number[] { + return protoBytes(20, [...protoText(1, "last_step_index"), ...protoText(2, String(index))]); +} + const TABLES = "CREATE TABLE gen_metadata (idx INTEGER, data BLOB); CREATE TABLE steps (idx INTEGER, metadata BLOB)"; @@ -130,7 +134,7 @@ describe("readAntigravityUsage", () => { }).pipe(Effect.provide(NodeServices.layer)), ); - it.effect("keeps explicit model evidence separate from retries and conflicting names", () => + it.effect("keeps explicit model evidence separate from conflicting names", () => Effect.gen(function* () { const path = yield* Path.Path; const dir = yield* tempDir; @@ -149,10 +153,6 @@ describe("readAntigravityUsage", () => { ...protoNumber(2, 10), ...protoText(11, `r-${idx}`), ]), - ...protoBytes( - 17, - protoBytes(2, [...protoNumber(1, 1026), ...protoNumber(2, 12)]), - ), ]), ), ]); @@ -173,15 +173,6 @@ describe("readAntigravityUsage", () => { 2, new Uint8Array(protoBytes(9, [...protoNumber(1, 1319), ...protoNumber(2, 10)])), ]); - yield* db.run("INSERT INTO gen_metadata VALUES (?, ?)", [ - 3, - new Uint8Array( - protoBytes(1, [ - ...protoText(19, "Gemini 9.3 Flash"), - ...protoBytes(17, protoBytes(2, [...protoNumber(1, 9004), ...protoNumber(2, 10)])), - ]), - ), - ]); yield* db.run("INSERT INTO steps VALUES (?, ?)", [ 4, new Uint8Array([ @@ -201,78 +192,98 @@ describe("readAntigravityUsage", () => { "gemini-9.1-flash", "antigravity-model-9003", "antigravity-model-1319", - "claude-opus-4-6", "gemini-9.2-flash", - "claude-opus-4-6", - "antigravity-model-9004", "claude-opus-5-5-thinking", ].sort(), ); }).pipe(Effect.provide(NodeServices.layer)), ); - it.effect( - "deduplicates Antigravity generation and step usage while preserving retry model and token buckets", - () => - Effect.gen(function* () { - const path = yield* Path.Path; - const dir = yield* tempDir; - const stamp = protoNumber(1, 1780000000); - const usage = [ - ...protoNumber(2, 100), - ...protoNumber(3, 40), - ...protoNumber(4, 5), - ...protoNumber(5, 20), - ...protoNumber(9, 10), - ...protoText(11, "response-1"), - ]; - const retry = [ - ...protoNumber(1, 1026), - ...protoNumber(2, 12), - ...protoNumber(3, 3), - ...protoText(11, "retry-1"), - ]; - const generation = protoBytes(1, [ - ...protoBytes(4, usage), - ...protoText(19, "Gemini 3 Pro"), - ...protoBytes(9, protoBytes(4, stamp)), - ]); - const step = [ - ...protoBytes(9, usage), - ...protoBytes(8, stamp), - ...protoBytes(28, protoBytes(2, retry)), - ]; - yield* seedDatabase(path.join(dir, "session-1.db"), (db) => - Effect.gen(function* () { - yield* createTables(db); - yield* db.run("INSERT INTO gen_metadata VALUES (?, ?)", [ - 0, - new Uint8Array(generation), - ]); - yield* db.run("INSERT INTO steps VALUES (?, ?)", [0, new Uint8Array(step)]); - }), - ); - const result = yield* readAntigravityUsage(dir, 0); - assert.deepStrictEqual(result.errors, []); - const read = records(result); - assert.strictEqual(read.length, 2); - const main = read.find((record) => record.model === "gemini-3-pro"); - assert.isDefined(main); - assert.strictEqual(main?.timestampMs, 1780000000000); - assert.strictEqual(main?.sessionId, "session-1"); - assert.deepStrictEqual(main?.totals, { - uncachedInputTokens: 100, - cachedInputTokens: 20, - cacheCreationTokens: 5, - outputTokens: 40, - reasoningTokens: 10, - }); - assert.strictEqual( - read.find((record) => record.model === "claude-opus-4-6")?.totals.uncachedInputTokens, - 12, - ); - assert.deepStrictEqual(records(yield* readAntigravityUsage(dir, 1780000000001)), []); - }).pipe(Effect.provide(NodeServices.layer)), + it.effect("counts each Antigravity generation once across its attempt list and step", () => + Effect.gen(function* () { + const path = yield* Path.Path; + const dir = yield* tempDir; + const stamp = protoNumber(1, 1780000000); + const usage = [ + ...protoNumber(2, 100), + ...protoNumber(3, 40), + ...protoNumber(4, 5), + ...protoNumber(5, 20), + ...protoNumber(9, 10), + ]; + yield* seedDatabase(path.join(dir, "session-1.db"), (db) => + Effect.gen(function* () { + yield* createTables(db); + yield* db.run("INSERT INTO gen_metadata VALUES (?, ?)", [ + 0, + new Uint8Array( + protoBytes(1, [ + ...protoBytes(4, usage), + ...protoText(19, "Gemini 3 Pro"), + ...protoBytes(9, protoBytes(4, stamp)), + ...protoBytes(17, protoBytes(2, usage)), + // Malformed key/value entries before the link must not hide it or fail the database. + ...protoBytes(20, [0x0a, 0x05]), + ...protoBytes(20, protoBytes(1, [0xff, 0xfe])), + ...lastStepIndex(0), + ]), + ), + ]); + yield* db.run("INSERT INTO steps VALUES (?, ?)", [ + 1, + new Uint8Array([...protoBytes(9, usage), ...protoBytes(8, stamp)]), + ]); + // A step without a generation row whose usage sums a failed attempt and its retry. + yield* db.run("INSERT INTO steps VALUES (?, ?)", [ + 2, + new Uint8Array([ + ...protoBytes(9, [...protoNumber(2, 12), ...protoNumber(3, 3)]), + ...protoBytes(8, stamp), + ...protoBytes(28, protoBytes(2, [...protoNumber(2, 7), ...protoNumber(3, 1)])), + ...protoBytes(28, protoBytes(2, [...protoNumber(2, 5), ...protoNumber(3, 2)])), + ]), + ]); + }), + ); + const result = yield* readAntigravityUsage(dir, 0); + assert.deepStrictEqual(result.errors, []); + const read = records(result); + assert.deepStrictEqual( + read.map(({ model, sessionId, timestampMs, totals }) => [ + model, + sessionId, + timestampMs, + totals, + ]), + [ + [ + "antigravity-unknown", + "session-1", + 1780000000000, + { + uncachedInputTokens: 12, + cachedInputTokens: 0, + cacheCreationTokens: 0, + outputTokens: 3, + reasoningTokens: 0, + }, + ], + [ + "gemini-3-pro", + "session-1", + 1780000000000, + { + uncachedInputTokens: 100, + cachedInputTokens: 20, + cacheCreationTokens: 5, + outputTokens: 40, + reasoningTokens: 10, + }, + ], + ], + ); + assert.deepStrictEqual(records(yield* readAntigravityUsage(dir, 1780000000001)), []); + }).pipe(Effect.provide(NodeServices.layer)), ); it.effect("uses the matching Antigravity generation model for each model-less step", () => @@ -285,10 +296,10 @@ describe("readAntigravityUsage", () => { for (const [idx, name] of ["Gemini 3 Pro", "Claude Opus 4.6"].entries()) { yield* db.run("INSERT INTO gen_metadata VALUES (?, ?)", [ idx, - new Uint8Array(protoBytes(1, protoText(19, name))), + new Uint8Array(protoBytes(1, [...protoText(19, name), ...lastStepIndex(idx * 2)])), ]); yield* db.run("INSERT INTO steps VALUES (?, ?)", [ - idx, + idx * 2 + 1, new Uint8Array(protoBytes(9, protoNumber(2, 10 + idx))), ]); } @@ -312,18 +323,20 @@ describe("readAntigravityUsage", () => { yield* createTables(db); yield* db.run("INSERT INTO gen_metadata VALUES (?, ?)", [ 0, - new Uint8Array(protoBytes(1, protoText(19, "Gemini 3.7 Flash Control"))), + new Uint8Array( + protoBytes(1, [...protoText(19, "Gemini 3.7 Flash Control"), ...lastStepIndex(0)]), + ), ]); yield* db.run("INSERT INTO steps VALUES (?, ?)", [ - 0, + 1, new Uint8Array(protoBytes(9, protoNumber(2, 10))), ]); yield* db.run("INSERT INTO steps VALUES (?, ?)", [ - 1, + 2, new Uint8Array(protoBytes(9, [...protoNumber(1, 1319), ...protoNumber(2, 11)])), ]); yield* db.run("INSERT INTO steps VALUES (?, ?)", [ - 2, + 3, new Uint8Array(protoBytes(9, [...protoNumber(1, 1132), ...protoNumber(2, 12)])), ]); // A model-less step that only learns its model by merging with a named generation. @@ -386,10 +399,10 @@ describe("readAntigravityUsage", () => { ].entries()) { yield* db.run("INSERT INTO gen_metadata VALUES (?, ?)", [ idx, - new Uint8Array(protoBytes(1, protoText(19, name))), + new Uint8Array(protoBytes(1, [...protoText(19, name), ...lastStepIndex(idx * 2)])), ]); yield* db.run("INSERT INTO steps VALUES (?, ?)", [ - idx, + idx * 2 + 1, new Uint8Array(protoBytes(9, protoNumber(2, 10 + idx))), ]); } diff --git a/apps/server/src/provider/Drivers/antigravityUsageReader.ts b/apps/server/src/provider/Drivers/antigravityUsageReader.ts index 4d473f50c9d67..25f8b4ef488e8 100644 --- a/apps/server/src/provider/Drivers/antigravityUsageReader.ts +++ b/apps/server/src/provider/Drivers/antigravityUsageReader.ts @@ -176,9 +176,11 @@ interface Metadata { model: string; modelId: number; timestampMs: number | null; - usages: Fields[]; + usage: Fields | undefined; + stepIndex: number | undefined; } +// Usage in fields 4 and 9 sums every attempt; the per-attempt lists in fields 17 and 28 repeat it. function metadata(bytes: Uint8Array, step: boolean): Metadata { const root = fields(bytes); if (!step && bytesAt(root, 1) === undefined) { @@ -186,28 +188,38 @@ function metadata(bytes: Uint8Array, step: boolean): Metadata { } const data = step ? root : nested(root, 1); const model = step ? nested(data, 24) : data; - const usage = bytesAt(data, step ? 9 : 4); - // Copy the usage bytes: subarrays kept in usages would hold each row's whole blob alive. - const primaryUsage = usage === undefined ? undefined : fields(usage.slice()); - const usages = primaryUsage === undefined ? [] : [primaryUsage]; - for (const retry of data.get(step ? 28 : 17) ?? []) { - if (!(retry instanceof Uint8Array)) throw new Error("Invalid retry metadata"); - const retryUsage = bytesAt(fields(retry), 2); - if (retryUsage !== undefined) usages.push(fields(retryUsage.slice())); - } + const usageBytes = bytesAt(data, step ? 9 : 4); + // Copy the usage bytes: a subarray would hold the row's whole blob alive. + const usage = usageBytes === undefined ? undefined : fields(usageBytes.slice()); const names = [textAt(model, step ? 12 : 19), textAt(model, step ? 8 : 21)].map(modelName); return { model: names.find(isConcreteModel) ?? names.find((name) => name && !/^model[_-]/.test(name)) ?? "", - modelId: - numberAt(model, step ? 1 : 3) || (primaryUsage === undefined ? 0 : numberAt(primaryUsage, 1)), + modelId: numberAt(model, step ? 1 : 3) || (usage === undefined ? 0 : numberAt(usage, 1)), timestampMs: step ? (timestamp(nested(data, 8)) ?? timestamp(nested(data, 1))) : timestamp(nested(nested(data, 9), 4)), - usages, + usage, + stepIndex: step ? undefined : linkedStepIndex(data), }; } +// A generation's usage is repeated on the step after its last_step_index. The link only merges that +// copy, so a malformed entry leaves the generation unlinked instead of failing the whole database. +function linkedStepIndex(data: Fields): number | undefined { + for (const entry of data.get(20) ?? []) { + try { + const pair = fields(blob(entry)); + if (textAt(pair, 1) !== "last_step_index") continue; + const index = textAt(pair, 2); + return /^\d+$/.test(index) ? Number(index) + 1 : undefined; + } catch { + continue; + } + } + return undefined; +} + function blob(value: unknown): Uint8Array { if (!(value instanceof Uint8Array)) throw new Error("Invalid Antigravity metadata blob"); return value; @@ -338,61 +350,69 @@ function usageCandidates( fallbackTimestamp: number, ): UsageCandidate[] { const records: UsageCandidate[] = []; - const generationModels = new Map(generations.map(({ idx, entry }) => [idx, entry])); + const generationsByStep = new Map( + generations.flatMap(({ entry }) => + entry.stepIndex === undefined ? [] : [[entry.stepIndex, entry] as const], + ), + ); for (const [source, entries] of [ ["step", steps], ["generation", generations], ] as const) { for (const [index, { idx, entry }] of entries.entries()) { - for (const [usageIndex, usage] of entry.usages.entries()) { - const modelId = numberAt(usage, 1) || entry.modelId; - const generation = source === "step" ? generationModels.get(idx) : undefined; - // A retry can use a different model than its enclosing generation or step. - const matches = (metadata: Metadata) => modelId === 0 || metadata.modelId === modelId; - const model = - matches(entry) && entry.model - ? entry.model - : generation && matches(generation) - ? generation.model - : ""; - const outputTokens = Math.max(numberAt(usage, 3), numberAt(usage, 9) + numberAt(usage, 10)); - const totals = { - uncachedInputTokens: numberAt(usage, 2), - cachedInputTokens: numberAt(usage, 5), - cacheCreationTokens: numberAt(usage, 4), - outputTokens, - reasoningTokens: Math.min(outputTokens, numberAt(usage, 9)), - }; - if ( - totals.uncachedInputTokens + - totals.cachedInputTokens + - totals.cacheCreationTokens + - outputTokens === - 0 - ) - continue; - const keys = ([11, 12, 7] as const).flatMap((key) => { - const id = textAt(usage, key); - return id ? [`antigravity:${key}:${id}`] : []; - }); - const record: UsageRecord = { - provider: "antigravity", - sessionId, - timestampMs: entry.timestampMs ?? trajectoryTimestamp ?? fallbackTimestamp, - model, - totals, - reportedCostUsd: null, - speed: "standard", - dedupeKey: keys[0] ?? `antigravity:${sessionId}:${source}:${index}:${usageIndex}`, - }; - records.push({ - record, - modelId, - modelQuality: isConcreteModel(model) ? 4 : model ? 1 : 0, - keys, - timestampQuality: entry.timestampMs !== null ? 2 : trajectoryTimestamp !== null ? 1 : 0, - }); - } + const usage = entry.usage; + if (usage === undefined) continue; + const modelId = numberAt(usage, 1) || entry.modelId; + const generation = source === "step" ? generationsByStep.get(idx) : undefined; + const matches = (metadata: Metadata) => modelId === 0 || metadata.modelId === modelId; + const model = + matches(entry) && entry.model + ? entry.model + : generation && matches(generation) + ? generation.model + : ""; + const outputTokens = Math.max(numberAt(usage, 3), numberAt(usage, 9) + numberAt(usage, 10)); + const totals = { + uncachedInputTokens: numberAt(usage, 2), + cachedInputTokens: numberAt(usage, 5), + cacheCreationTokens: numberAt(usage, 4), + outputTokens, + reasoningTokens: Math.min(outputTokens, numberAt(usage, 9)), + }; + if ( + totals.uncachedInputTokens + + totals.cachedInputTokens + + totals.cacheCreationTokens + + outputTokens === + 0 + ) + continue; + const stepIndex = source === "step" ? idx : entry.stepIndex; + const responseKeys = ([11, 12, 7] as const).flatMap((key) => { + const id = textAt(usage, key); + return id ? [`antigravity:${key}:${id}`] : []; + }); + const identity = + stepIndex === undefined + ? `antigravity:${sessionId}:generation:${index}` + : `antigravity:${sessionId}:step:${stepIndex}`; + const record: UsageRecord = { + provider: "antigravity", + sessionId, + timestampMs: entry.timestampMs ?? trajectoryTimestamp ?? fallbackTimestamp, + model, + totals, + reportedCostUsd: null, + speed: "standard", + dedupeKey: responseKeys[0] ?? identity, + }; + records.push({ + record, + modelId, + modelQuality: isConcreteModel(model) ? 4 : model ? 1 : 0, + keys: [...responseKeys, identity], + timestampQuality: entry.timestampMs !== null ? 2 : trajectoryTimestamp !== null ? 1 : 0, + }); } } return records;