diff --git a/CHANGELOG.md b/CHANGELOG.md index 55aac5fe3..4a43a091b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,10 @@ ## Unreleased +### Fixed +- **Copilot agent requests in VS Code chat count every model call.** Copilot Chat 0.69 keeps only the last call of an agent request in its metadata, so CodeBurn showed a fraction of the output and cost. It now reads the request's totals (all calls, with the cache split when VS Code records it), and a request that records GitHub's credits costs exactly that bill (1 credit = $0.01) in reports and plan credits. A chat session that also ran on the agent host is counted once, from the agent host. In Copilot plan credits each chat request is rated on its own. Past Copilot days re-derive once after the update. +- Copilot showed `claude-sonnet-4-6` as "Sonnet 4" instead of "Sonnet 4.6". + ## 0.9.27 - 2026-10-11 ### Added diff --git a/docs/providers/copilot.md b/docs/providers/copilot.md index b5ccba193..35a825624 100644 --- a/docs/providers/copilot.md +++ b/docs/providers/copilot.md @@ -28,7 +28,16 @@ Microsoft 365 Copilot (Office, Teams) and Copilot in the browser are not support JSONL in the first three locations (schemas differ; the parser switches by source type / event shape), a SQLite DB for the OTel source, and a Nitrite (H2 MVStore) `.db` for the JetBrains source. VS Code core chat sessions use a delta journal: `kind:0` sets the root object, `kind:1` writes a value at path `k`, and `kind:2` appends items to an array path. -Core chat-session journals read input from `result.metadata.promptTokens`, falling back to the request's `promptTokens` when metadata has no positive count. Output keeps the existing `result.metadata.outputTokens` then request `completionTokens` precedence. These fields are alternatives, never summed. Input-only requests count; rows without reported usage remain skipped rather than estimated from text. The request-level prompt count is the value VS Code recorded, not a reconstructed total across an agent loop's model calls. +An agent-mode request makes one model call per tool round. In Copilot Chat 0.69 (VS Code 1.141) `result.metadata.promptTokens` and `outputTokens` describe only the last of those calls, while the request itself carries the whole request: `completionTokens` (output summed over every call), `modelTotals` (per model `inputTokens`, `cachedTokens`, `outputTokens`, summed over every call) and `copilotCredits` (GitHub's billed `total_nano_aiu / 1e9`, subagents included). Core chat sessions, journal and legacy JSON alike, read each request in this order: + +1. `modelTotals`, summed over its models: output, cache reads from `cachedTokens`, and input without the cached part (the Copilot engine counts cached tokens inside `inputTokens`; an input below the cached count is taken as already uncached). +2. Otherwise output from the request's `completionTokens`, then `result.metadata.outputTokens`; input from `result.metadata.promptTokens`, then the request's `promptTokens`. Both prompt fields hold the last call only, and there is no cache split, so input of a multi-round request without `modelTotals` stays undercounted. + +The fields are alternatives, never summed. `copilotCredits` rides as `nanoAiu`, exactly like the agent host's `totalNanoAiu`, so plan credits for these requests are exact, and it sets the request's cost to the bill (credits × $0.01, kept through the cache as a billed cost) because the last-round prompt cannot price it. Requests without `copilotCredits` stay priced from the tokens above. VS Code fills `copilotCredits` from GitHub's `total_nano_aiu` on the extension path, but for an agent-host turn without that figure it falls back to the agent's own `_meta.cost`. The saved request does not say which one it holds. VS Code 1.141 saves full requests only for local sessions, which take the `total_nano_aiu` path, so the figure is treated as the bill; a saved request from the fallback would be priced as if it were. + +A chat session that also ran on the agent host is counted once, from the agent host. At serve time, a journal request is left out when a session with exactly the same id has billed usage on the agent-host side: a session-state `events.jsonl` call with tokens or `nanoAiu`, or a session-store row. A session-state file with no usage yet keeps the journal. VS Code 1.141 does not save agent-host sessions as full journals (only an index entry), so this guards a shape not seen on real data. + +In plan credits, a chat-session request is rated on its own: a request with `copilotCredits` counts as exact, one without keeps its token estimate, so a session spanning the Copilot Chat 0.69 upgrade is not billed whole. `sessionCopilotCredits` is a running session total and is not read. `result.metadata.summaries[]` holds only compaction text (`toolCallRoundId`, `text`); compaction usage reaches the request's own usage fields, so it adds nothing. Input-only and credits-only requests count; rows without reported usage remain skipped rather than estimated from text. ## VS Code agent host (Copilot CLI engine) @@ -197,9 +206,10 @@ see the #927 ruling in `src/session-cache.ts`). Copilot AI credits (`codeburn plan set copilot-pro`). Billing-grade cost rewrite of every report is still upstream #890. - These CLI session-store rows and the `session.shutdown` rollup's - `totalNanoAiu` (CLI and VS Code agent host) are the only local sources that - carry an exact credit figure. VS Code chat sessions and transcripts, the OTel + These CLI session-store rows, the `session.shutdown` rollup's + `totalNanoAiu` (CLI and VS Code agent host) and `copilotCredits` on VS Code + chat-session requests (Copilot Chat 0.69 on) are the only local sources that + carry an exact credit figure. Older chat sessions, transcripts, the OTel `agent-traces.db` and JetBrains stores never do, so on a typical machine most requests have no exact figure at all (#1199). Everything unrated is estimated instead: the request's tokens priced at the @@ -209,7 +219,7 @@ see the #927 ruling in `src/session-cache.ts`). (the legacy premium-request weight) never touches the credit figure. So the estimate's accuracy hinges on the source: on Copilot CLI events, which carry the cache split, CodeBurn's token pricing landed within about 15% of the - exact bill; VS Code chat sessions record prompt and output tokens only, so + exact bill; older VS Code chat sessions record prompt and output tokens only, so their estimate can land either side of the bill, and the numbers in #1199 show the gap can reach roughly 2x. A session that carries any exact figure is never estimated on top of, because `total_nano_aiu` already bills that diff --git a/src/daily-cache.ts b/src/daily-cache.ts index 7762dd816..3ded9cc7b 100644 --- a/src/daily-cache.ts +++ b/src/daily-cache.ts @@ -357,7 +357,11 @@ import type { DateRange, ProjectSummary } from './types.js' // instead of dropped, Gemini 3.8 Flash placeholders get their own ids and // Gemini 3 Flash -a/-d rows are priced. Calls and cost only rise, so no // PENDING_REDERIVE_PROVIDER_VERSIONS entry is needed. -export const DAILY_CACHE_VERSION = 74 +// v75: Copilot chat-session requests count every agent round and bill from +// copilotCredits, and a chat session that also ran on the agent host is left +// to it. That twin drops calls, so copilot joins +// PENDING_REDERIVE_PROVIDER_VERSIONS at 75. +export const DAILY_CACHE_VERSION = 75 const MIN_SUPPORTED_VERSION = 28 /// Providers whose per-day CALL COUNT means something different at @@ -382,8 +386,9 @@ const MIN_SUPPORTED_VERSION = 28 /// untouched, in both directions, and every other provider keeps the guard. const PENDING_REDERIVE_PROVIDER_VERSIONS: Readonly> = { // 56: a store row now pairs with its tokenless per-turn twin, which can - // move one call across midnight. - copilot: 56, + // move one call across midnight. 75: chat sessions that also ran on the + // agent host are no longer counted from the journal. + copilot: 75, // Codex response records replace stale/zero token_count twins and can // legitimately reduce counts as well as recover missing usage. codex: 43, diff --git a/src/models.ts b/src/models.ts index 125ccc1dc..1ee7b99de 100644 --- a/src/models.ts +++ b/src/models.ts @@ -1758,10 +1758,11 @@ function lookupShortName(id: string): string | undefined { /// so `gpt-5` cannot capture `gpt-5.5` or `gpt-5.6-luna`, exactly as the /// global table's `id === key || id.startsWith(key + '-')` cannot (#1530). /// Providers match their local tables with this so one model id resolves to -/// one display name on every provider. +/// one display name on every provider. A one- or two-digit dash segment is a +/// minor version (`claude-sonnet-4-6`), not a suffix; dates are longer. export function modelKeyMatches(model: string, key: string): boolean { const escaped = key.replace(/[.*+?^${}()|[\]\\]/g, '\\$&') - return new RegExp(`(? nanRollupFallbackTs: Map sessionEarliestValidTs: Map + agentHostUsageSessions: Set } | null = null if (providerName === 'copilot') { // DEPENDS on the durable full-load exemption in session-cache.ts @@ -4189,6 +4190,11 @@ export async function parseProviderSources( // sealed days. const nanRollupFallbackTs = new Map() const sessionEarliestValidTs = new Map() + // Sessions the agent host bills (session-state rollups or turns, store + // rows). A VS Code chat-session journal of the same session is a second + // record of the same requests, so it is left out at serve time. A + // session-state file with no usage yet keeps the journal. + const agentHostUsageSessions = new Set() // Session-state per-turn calls carry no per-call project (the serve loop // takes it from source.project), so resolve it the same way here — every // call of the session must serve under the SAME label, whichever of the @@ -4203,6 +4209,10 @@ export async function parseProviderSources( seenAggKeys.add(c.deduplicationKey) const ts = new Date(c.timestamp).getTime() const isStore = c.deduplicationKey.startsWith('copilot-store:') + if ((isStore || basename(cachedPath) === 'events.jsonl') && (c.nanoAiu != null || + c.usage.inputTokens + c.usage.outputTokens + c.usage.cacheReadInputTokens + c.usage.cacheCreationInputTokens > 0)) { + agentHostUsageSessions.add(turn.sessionId) + } const isRollup = c.deduplicationKey.startsWith(shutdownPrefix) const aggKey = `${turn.sessionId}\n${c.model}` if (!Number.isNaN(ts)) { @@ -4306,7 +4316,7 @@ export async function parseProviderSources( } } } - copilotRecon = { storeKeys, storeCalls, rollupLegs, supplementaryStoreKeys, turnOwnedOutputKeys, sessionProject, storeProject, nanRollupFallbackTs, sessionEarliestValidTs } + copilotRecon = { storeKeys, storeCalls, rollupLegs, supplementaryStoreKeys, turnOwnedOutputKeys, sessionProject, storeProject, nanRollupFallbackTs, sessionEarliestValidTs, agentHostUsageSessions } } const copilotServeProject = (sessionId: string): string | undefined => copilotRecon @@ -4318,6 +4328,10 @@ export async function parseProviderSources( let changed = false const kept: CachedCall[] = [] for (const c of turn.calls) { + if (c.deduplicationKey.startsWith('copilot-chatsession:') && copilotRecon.agentHostUsageSessions.has(turn.sessionId)) { + changed = true + continue + } if (c.deduplicationKey.startsWith(shutdownPrefix)) { const tsValid = !Number.isNaN(new Date(c.timestamp).getTime()) if (tsValid && copilotRecon.storeKeys.has(`${turn.sessionId}\n${c.model}`)) { diff --git a/src/plan-usage.ts b/src/plan-usage.ts index 23e00dce3..dd97770f1 100644 --- a/src/plan-usage.ts +++ b/src/plan-usage.ts @@ -117,7 +117,17 @@ export function copilotCreditSpend(projects: ProjectSummary[]): CopilotCreditSpe // supplementary twin across a local-day boundary, so both nanoAiu rows can // occupy separate turns of the same session. Close the twin once here. forEachPlanSession(projects, calls => { - const copilot = calls.filter(call => call.provider === 'copilot') + // A VS Code chat-session request's copilotCredits bills that request + // alone, so a session that spans the Copilot Chat 0.69 upgrade keeps an + // estimate for its older requests. Agent-host figures bill the session. + const isChatRequest = (call: ParsedApiCall) => call.deduplicationKey.startsWith('copilot-chatsession:') + const chatRated = calls.filter(call => call.provider === 'copilot' && isChatRequest(call) && isFiniteNanoAiu(call.nanoAiu)) + const chatUnrated = calls.filter(call => call.provider === 'copilot' && isChatRequest(call) && !isFiniteNanoAiu(call.nanoAiu)) + for (const call of chatRated) nanoSum += call.nanoAiu! + for (const call of chatUnrated) estimatedUsd += call.costUSD + ratedCalls += behavioralCallCount(chatRated) + unratedCalls += behavioralCallCount(chatUnrated) + const copilot = calls.filter(call => call.provider === 'copilot' && !isChatRequest(call)) if (copilot.length === 0) return const primaryNano = copilot.filter(call => !call.supplementaryAccounting && isFiniteNanoAiu(call.nanoAiu)) const suppNano = copilot.filter(call => call.supplementaryAccounting && isFiniteNanoAiu(call.nanoAiu)) @@ -161,7 +171,7 @@ export function copilotCreditsNote(rated: number, unrated: number): string { if (unrated === 0) { return `All ${rated} Copilot requests in this period carry GitHub's exact credit figure, so spentCredits is complete.` } - return `${rated} of ${rated + unrated} Copilot requests carry GitHub's exact credit figure (only Copilot CLI and VS Code agent-host sessions record it). spentCredits counts only those; estimatedCredits adds the rest priced from tokens at listed API rates, which matched GitHub within about 15% on real Copilot CLI events; VS Code chat sessions record no cache split, so their estimate can land either side of the bill. percentUsed tracks estimatedCredits while any request is unrated.` + return `${rated} of ${rated + unrated} Copilot requests carry GitHub's exact credit figure (Copilot CLI, VS Code agent-host sessions and VS Code chat sessions from Copilot Chat 0.69 on record it). spentCredits counts only those; estimatedCredits adds the rest priced from tokens at listed API rates, which matched GitHub within about 15% on real Copilot CLI events; older VS Code chat sessions record no cache split and only the last model call's prompt, so their estimate can land either side of the bill. percentUsed tracks estimatedCredits while any request is unrated.` } export function projectMonthEnd( diff --git a/src/providers/copilot.ts b/src/providers/copilot.ts index 168f75a3f..d43e0f631 100644 --- a/src/providers/copilot.ts +++ b/src/providers/copilot.ts @@ -66,6 +66,7 @@ import { getEditorDataDir } from '../editor-data-dir.js' import { createHash } from 'crypto' import { readSessionFile } from '../fs-utils.js' import { calculateCost, modelKeyMatches } from '../models.js' +import { NANO_AIU_PER_CREDIT, creditsToUsd, nanoAiuToCredits } from '../copilot-aiu.js' import { extractBashCommands } from '../bash-utils.js' import { estimateTokens } from '../context-tree.js' import type { @@ -202,6 +203,8 @@ interface LegacyChatRequest { promptTokens?: number completionTokens?: number outputTokens?: number + modelTotals?: unknown + copilotCredits?: unknown } interface LegacyChatSession { @@ -533,6 +536,35 @@ function extractModelFromRequest(req: LegacyChatRequest, session: LegacyChatSess return 'unknown' } +// An agent-mode request makes one model call per tool round, but +// result.metadata.promptTokens/outputTokens describe only the last one. +// modelTotals sums every round per model. Its inputTokens includes the cached +// part when it comes from the Copilot engine and excludes it when it comes from +// the Claude SDK; an input below the cached count can only be the latter. +function chatSessionModelTotals(raw: unknown): { inputTokens: number; cacheReadTokens: number; outputTokens: number } | undefined { + if (!Array.isArray(raw)) return undefined + let input = 0 + let cached = 0 + let output = 0 + let found = false + for (const entry of raw) { + if (!isRecord(entry)) continue + found = true + input += numberOrZero(entry['inputTokens']) + cached += numberOrZero(entry['cachedTokens']) + output += numberOrZero(entry['outputTokens']) + } + if (!found) return undefined + return { inputTokens: input >= cached ? input - cached : input, cacheReadTokens: cached, outputTokens: output } +} + +// copilotCredits is GitHub's billed total_nano_aiu / 1e9 for the whole request, +// subagents included, so it prices the request where the last-round prompt +// tokens cannot. +function chatSessionNanoAiu(raw: unknown): number | undefined { + return typeof raw === 'number' && Number.isFinite(raw) && raw >= 0 ? Math.round(raw * NANO_AIU_PER_CREDIT) : undefined +} + function parseLegacyChatSession( session: LegacyChatSession, sessionId: string, @@ -548,25 +580,31 @@ function parseLegacyChatSession( const meta = isRecord(resultObj?.['metadata']) ? (resultObj?.['metadata'] as Record) : null const usage = isRecord(resultObj?.['usage']) ? (resultObj?.['usage'] as Record) : null + const totals = chatSessionModelTotals(req.modelTotals) + const nanoAiu = chatSessionNanoAiu(req.copilotCredits) + // Exact token extraction: check metadata, result, root req, and usage independently - const exactPromptTokens = + const exactPromptTokens = totals ? totals.inputTokens : ( numberOrZero(meta?.['promptTokens']) || numberOrZero(resultObj?.['promptTokens']) || numberOrZero(req.promptTokens) || numberOrZero(usage?.['promptTokens']) || numberOrZero(usage?.['prompt_tokens']) || undefined + ) - const exactOutputTokens = + // Root completionTokens is summed over every round; metadata holds the last. + const exactOutputTokens = totals ? totals.outputTokens : ( + numberOrZero(req.completionTokens) || numberOrZero(meta?.['outputTokens']) || numberOrZero(resultObj?.['outputTokens']) || - numberOrZero(req.completionTokens) || numberOrZero(req.outputTokens) || numberOrZero(usage?.['completionTokens']) || numberOrZero(usage?.['completion_tokens']) || undefined + ) - const exactCacheRead = + const exactCacheRead = totals ? totals.cacheReadTokens : numberOrZero(meta?.['cacheReadTokens']) || numberOrZero(meta?.['cachedTokens']) || numberOrZero(resultObj?.['cacheReadTokens']) || @@ -577,7 +615,7 @@ function parseLegacyChatSession( numberOrZero(usage?.['cached_tokens']) || 0 - const exactCacheCreation = + const exactCacheCreation = totals ? 0 : numberOrZero(meta?.['cacheCreationTokens']) || numberOrZero(meta?.['cacheWriteTokens']) || numberOrZero(resultObj?.['cacheCreationTokens']) || @@ -684,7 +722,7 @@ function parseLegacyChatSession( if (seenKeys.has(dedupKey)) continue seenKeys.add(dedupKey) - const costUSD = calculateCost( + const costUSD = nanoAiu !== undefined ? creditsToUsd(nanoAiuToCredits(nanoAiu)) : calculateCost( model, totalInputTokens, outputTokens + reasoningTokens, @@ -706,7 +744,7 @@ function parseLegacyChatSession( reasoningTokens, webSearchRequests: 0, costUSD, - costIsEstimated: isEstimated, + costIsEstimated: isEstimated && nanoAiu === undefined, tools, bashCommands: extracted.bashCommands, skills: extracted.skills.length > 0 ? extracted.skills : undefined, @@ -716,6 +754,7 @@ function parseLegacyChatSession( userMessage: msgText, sessionId, project, + ...(nanoAiu !== undefined ? { nanoAiu, costFromBilling: true } : {}), }) } @@ -1795,13 +1834,15 @@ function createChatSessionParser( const rawMetadata = resultRecord?.['metadata'] const metadata = isRecord(rawMetadata) ? rawMetadata : createReplayObject() - // VS Code also serializes usage on the request itself. Keep the - // extension metadata's precedence, as in the legacy JSON parser. - const inputTokens = numberOrZero(metadata['promptTokens']) || numberOrZero(rawReq['promptTokens']) - const metadataOutputTokens = numberOrZero(metadata['outputTokens']) - const outputTokens = metadataOutputTokens || numberOrZero(rawReq['completionTokens']) + // VS Code also serializes usage on the request itself, with output + // summed over every round. Same precedence as the legacy JSON parser. + const totals = chatSessionModelTotals(rawReq['modelTotals']) + const nanoAiu = chatSessionNanoAiu(rawReq['copilotCredits']) + const inputTokens = totals ? totals.inputTokens : numberOrZero(metadata['promptTokens']) || numberOrZero(rawReq['promptTokens']) + const outputTokens = totals ? totals.outputTokens : numberOrZero(rawReq['completionTokens']) || numberOrZero(metadata['outputTokens']) + const cacheReadTokens = totals?.cacheReadTokens ?? 0 - if (inputTokens === 0 && outputTokens === 0) continue + if (inputTokens === 0 && outputTokens === 0 && cacheReadTokens === 0 && !nanoAiu) continue const requestId = readString(rawReq['requestId']) || `request-${index}` const dedupKey = `copilot-chatsession:${sessionId}:${requestId}` @@ -1809,7 +1850,9 @@ function createChatSessionParser( seenKeys.add(dedupKey) const model = modelFromChatSessionRequest(rawReq, metadata) - const costUSD = calculateCost(model, inputTokens, outputTokens, 0, 0, 0) + const costUSD = nanoAiu !== undefined + ? creditsToUsd(nanoAiuToCredits(nanoAiu)) + : calculateCost(model, inputTokens, outputTokens, 0, cacheReadTokens, 0) const timestamp = timestampToISO(rawReq['timestamp']) || sessionCreatedAt const userMessage = isRecord(rawReq['message']) ? readString(rawReq['message']['text']) : '' @@ -1823,7 +1866,7 @@ function createChatSessionParser( inputTokens, outputTokens, cacheCreationInputTokens: 0, - cacheReadInputTokens: 0, + cacheReadInputTokens: cacheReadTokens, cachedInputTokens: 0, reasoningTokens: 0, webSearchRequests: 0, @@ -1835,6 +1878,7 @@ function createChatSessionParser( speed: 'standard' as const, deduplicationKey: dedupKey, userMessage, + ...(nanoAiu !== undefined ? { nanoAiu, costFromBilling: true } : {}), } } }, diff --git a/src/session-cache.ts b/src/session-cache.ts index ec9d59d5f..65c65ba4f 100644 --- a/src/session-cache.ts +++ b/src/session-cache.ts @@ -492,7 +492,9 @@ export const PROVIDER_PARSE_VERSIONS: Record = { // Keys are unchanged, so the re-parse replaces cached output-0 rows in place. // jetbrains-opus-4-7-v1: JetBrains claude-opus-4.7 turns priced as the // anthropic-auto fallback; cached entries must re-parse once. - copilot: 'cli-shutdown-cost-v1-skills-source-provenance-v1-session-store-v3-chatsession-otel-skills-v1-otel-trace-metadata-once-v1-transcript-unknown-usage-v1-otel-workspace-project-v1-journal-request-input-v1-tokenless-turns-v1-store-row-output-v1-jetbrains-opus-4-7-v1', + // chatsession-agent-totals-v1: chat-session requests read every agent round + // (modelTotals, root completionTokens) and copilotCredits as nanoAiu. + copilot: 'cli-shutdown-cost-v1-skills-source-provenance-v1-session-store-v3-chatsession-otel-skills-v1-otel-trace-metadata-once-v1-transcript-unknown-usage-v1-otel-workspace-project-v1-journal-request-input-v1-tokenless-turns-v1-store-row-output-v1-jetbrains-opus-4-7-v1-chatsession-agent-totals-v1', // authoritative-usage-v4: persist one Grok session call from top-level // authoritative totals, use modelUsage only for priced attribution, clamp // reasoning per record, and label mixed sessions estimated. diff --git a/tests/copilot-chatsession-twin.test.ts b/tests/copilot-chatsession-twin.test.ts new file mode 100644 index 000000000..592264bb2 --- /dev/null +++ b/tests/copilot-chatsession-twin.test.ts @@ -0,0 +1,120 @@ +// A VS Code chat session that also ran on the agent host is counted once, from +// the agent host. Decided at serve time over the whole cached Copilot set. + +import { afterEach, beforeEach, describe, expect, it } from 'vitest' +import { mkdir, mkdtemp, rm, writeFile } from 'fs/promises' +import { tmpdir } from 'os' +import { join } from 'path' +import { createRequire } from 'node:module' + +import { isSqliteAvailable } from '../src/sqlite.js' +import { clearLoadCacheMemo } from '../src/session-cache.js' +import { clearSessionCache, parseAllSessions } from '../src/parser.js' + +const requireForTest = createRequire(import.meta.url) +type TestDb = { exec(sql: string): void; prepare(sql: string): { run(...p: unknown[]): void }; close(): void } + +const ENV_KEYS = ['HOME', 'USERPROFILE', 'CODEBURN_CACHE_DIR', 'CODEBURN_COPILOT_SESSION_STORE_DB', 'CODEBURN_COPILOT_DISABLE_OTEL', + 'CODEBURN_COPILOT_SESSION_STATE_DIR', 'CODEBURN_COPILOT_WS_STORAGE_DIR', 'CODEBURN_COPILOT_GLOBAL_STORAGE_DIR', 'CODEBURN_COPILOT_JETBRAINS_DIR'] +let saved: Record +let tmp: string + +const ev = (type: string, timestamp: string, data: Record) => JSON.stringify({ type, timestamp, data }) + +async function writeJournal(sessionId: string) { + const dir = join(tmp, 'global', 'emptyWindowChatSessions') + await mkdir(dir, { recursive: true }) + await writeFile(join(dir, `${sessionId}.jsonl`), JSON.stringify({ kind: 0, v: { sessionId, creationDate: Date.parse('2026-08-05T12:00:00Z'), requests: [{ + requestId: 'r1', modelId: 'copilot/gpt-4o', timestamp: Date.parse('2026-08-05T12:00:00Z'), promptTokens: 1000, completionTokens: 50, copilotCredits: 2, + }] } }) + '\n') +} + +async function writeSessionState(sessionId: string, lines: string[]) { + const dir = join(tmp, 'session-state', sessionId) + await mkdir(dir, { recursive: true }) + await writeFile(join(dir, 'workspace.yaml'), `id: ${sessionId}\ncwd: /tmp/proj\n`) + await writeFile(join(dir, 'events.jsonl'), lines.join('\n') + '\n') +} + +function writeStore(sessionId: string) { + const { DatabaseSync } = requireForTest('node:sqlite') as { DatabaseSync: new (path: string) => TestDb } + const db = new DatabaseSync(join(tmp, 'session-store.db')) + db.exec(` + CREATE TABLE sessions (id TEXT PRIMARY KEY, cwd TEXT, repository TEXT, branch TEXT, created_at TEXT DEFAULT (datetime('now'))); + CREATE TABLE assistant_usage_events ( + id INTEGER PRIMARY KEY AUTOINCREMENT, session_id TEXT NOT NULL REFERENCES sessions(id), model TEXT NOT NULL, + input_tokens INTEGER, output_tokens INTEGER, cache_read_tokens INTEGER, cache_write_tokens INTEGER, reasoning_tokens INTEGER, + total_nano_aiu INTEGER, request_multiplier REAL, created_at TEXT DEFAULT (datetime('now')) + ); + `) + db.prepare(`INSERT INTO sessions (id, cwd) VALUES (?, '/tmp/proj')`).run(sessionId) + db.prepare(`INSERT INTO assistant_usage_events (session_id, model, input_tokens, output_tokens, cache_read_tokens, cache_write_tokens, reasoning_tokens, total_nano_aiu, created_at) + VALUES (?, 'gpt-4o', 1000, 50, 0, 0, 0, 2000000000, '2026-08-05T12:00:01.000Z')`).run(sessionId) + db.close() +} + +async function servedKeys(): Promise { + clearSessionCache() + const projects = await parseAllSessions(undefined, 'copilot') + return projects.flatMap(p => p.sessions).flatMap(s => s.turns).flatMap(t => t.assistantCalls).map(c => c.deduplicationKey).sort() +} + +beforeEach(async () => { + saved = Object.fromEntries(ENV_KEYS.map(k => [k, process.env[k]])) + tmp = await mkdtemp(join(tmpdir(), 'copilot-twin-')) + Object.assign(process.env, { + HOME: tmp, USERPROFILE: tmp, CODEBURN_CACHE_DIR: join(tmp, 'cache'), + CODEBURN_COPILOT_SESSION_STORE_DB: join(tmp, 'session-store.db'), CODEBURN_COPILOT_DISABLE_OTEL: '1', + CODEBURN_COPILOT_SESSION_STATE_DIR: join(tmp, 'session-state'), CODEBURN_COPILOT_WS_STORAGE_DIR: join(tmp, 'no-ws'), + CODEBURN_COPILOT_GLOBAL_STORAGE_DIR: join(tmp, 'global'), CODEBURN_COPILOT_JETBRAINS_DIR: join(tmp, 'no-jb'), + }) +}) + +afterEach(async () => { + clearSessionCache() + clearLoadCacheMemo() + for (const [k, v] of Object.entries(saved)) { + if (v === undefined) delete process.env[k] + else process.env[k] = v + } + await rm(tmp, { recursive: true, force: true }) +}) + +describe('copilot chat-session twins of agent-host sessions', () => { + it('drops the journal copy when a session-state rollup bills the session', async () => { + await writeJournal('twin') + await writeJournal('solo') + await writeSessionState('twin', [ + ev('session.start', '2026-08-05T11:59:00Z', { selectedModel: 'gpt-4o' }), + ev('assistant.message', '2026-08-05T12:00:00Z', { messageId: 'm1', model: 'gpt-4o', toolRequests: [] }), + ev('session.shutdown', '2026-08-05T12:01:00Z', { + shutdownType: 'routine', + modelMetrics: { 'gpt-4o': { usage: { inputTokens: 1000, outputTokens: 50, cacheReadTokens: 0, cacheWriteTokens: 0 }, totalNanoAiu: 2_000_000_000 } }, + }), + ]) + + const keys = await servedKeys() + expect(keys).toContain('copilot-chatsession:solo:r1') + expect(keys).not.toContain('copilot-chatsession:twin:r1') + expect(keys.some(k => k.startsWith('copilot:twin:shutdown:'))).toBe(true) + }) + + it('keeps the journal when the session-state file has no usage', async () => { + await writeJournal('pending') + await writeSessionState('pending', [ + ev('session.start', '2026-08-05T11:59:00Z', { selectedModel: 'gpt-4o' }), + ev('assistant.message', '2026-08-05T12:00:00Z', { messageId: 'm1', model: 'gpt-4o', toolRequests: [] }), + ]) + + expect(await servedKeys()).toContain('copilot-chatsession:pending:r1') + }) + + it.skipIf(!isSqliteAvailable())('drops the journal copy when only session-store rows bill the session', async () => { + await writeJournal('stored') + writeStore('stored') + + const keys = await servedKeys() + expect(keys).not.toContain('copilot-chatsession:stored:r1') + expect(keys.some(k => k.startsWith('copilot-store:stored:'))).toBe(true) + }) +}) diff --git a/tests/daily-cache-version-rederivation.test.ts b/tests/daily-cache-version-rederivation.test.ts index 413d3e426..ff94ac6ac 100644 --- a/tests/daily-cache-version-rederivation.test.ts +++ b/tests/daily-cache-version-rederivation.test.ts @@ -201,3 +201,33 @@ describe('cursor-agent re-derive after Cursor IDE chats moved to cursor (v73)', expect(prunedDay.cost).toBe(4) }) }) + +describe('copilot re-derive after chat-session twins moved to the agent host (v75)', () => { + it('replaces a settled copilot slice with fewer fresh calls and leaves other providers alone', async () => { + const settled = toDateString(new Date(Date.now() - 10 * 24 * 60 * 60 * 1000)) + const yesterday = toDateString(new Date(Date.now() - 24 * 60 * 60 * 1000)) + const slice = (calls: number, cost: number) => ({ ...day(settled, cost).providers.grok!, calls }) + const entry = (copilotCalls: number, copilotCost: number): DailyEntry => ({ + ...day(settled, copilotCost + 1), + calls: copilotCalls + 1, + providers: { copilot: slice(copilotCalls, copilotCost), claude: slice(1, 1) }, + models: {}, + }) + await writeFile(join(cacheRoot, 'daily-cache.v74.json'), JSON.stringify({ + version: 74, + savingsConfigHash: 'cfg', + tzKey: currentTzKey(), + lastComputedDate: yesterday, + days: [entry(5, 9)], + complete: true, + watermarkTrusted: true, + })) + + const hydrated = await ensureCacheHydrated(async () => [], () => [entry(3, 4)], 'cfg', () => true) + + const settledDay = hydrated.days.find(d => d.date === settled)! + expect(settledDay.providers.copilot).toMatchObject({ calls: 3, cost: 4 }) + expect(settledDay.providers.claude).toMatchObject({ calls: 1, cost: 1 }) + expect(hydrated.pendingRederive).toBeUndefined() + }) +}) diff --git a/tests/models.test.ts b/tests/models.test.ts index 177e36ff7..38e9e457a 100644 --- a/tests/models.test.ts +++ b/tests/models.test.ts @@ -766,6 +766,13 @@ describe('modelKeyMatches', () => { expect(modelKeyMatches('xgpt-5', 'gpt-5')).toBe(false) expect(modelKeyMatches('claude-opus-4.8', 'claude-opus-4')).toBe(false) }) + + it('treats a short numeric dash segment as a minor version, not a suffix', () => { + expect(modelKeyMatches('claude-sonnet-4-6', 'claude-sonnet-4')).toBe(false) + expect(modelKeyMatches('claude-sonnet-4-6-20260101', 'claude-sonnet-4')).toBe(false) + expect(modelKeyMatches('claude-sonnet-4-20250514', 'claude-sonnet-4')).toBe(true) + expect(modelKeyMatches('claude-sonnet-4-5', 'claude-sonnet-4-5')).toBe(true) + }) }) describe('claude-fable-5 pricing + name', () => { diff --git a/tests/plan-usage.test.ts b/tests/plan-usage.test.ts index 204d800fd..8277964da 100644 --- a/tests/plan-usage.test.ts +++ b/tests/plan-usage.test.ts @@ -627,6 +627,21 @@ describe('copilot AI credit plan math', () => { expect(usage.creditsIncomplete).toBe(false) }) + it('rates VS Code chat-session requests one by one in a session that spans the 0.69 upgrade', () => { + const chat = (ts: string, costUSD: number, nanoAiu?: number) => usageCall({ + provider: 'copilot', costUSD, timestamp: `2026-08-05T12:00:${ts}.000Z`, deduplicationKey: `copilot-chatsession:s:${ts}`, + ...(nanoAiu !== undefined ? { nanoAiu } : {}), + }) + const spend = copilotCreditSpend([usageProject([chat('00', 0.2), chat('01', 0.965, 96_500_000_000)])]) + + expect(spend.spentCredits).toBe(96.5) + expect(spend.estimatedCredits).toBeCloseTo(116.5, 9) + expect(spend.creditRatedCalls).toBe(1) + expect(spend.creditUnratedCalls).toBe(1) + expect(spend.creditsIncomplete).toBe(true) + expect(copilotCreditsNote(spend.creditRatedCalls, spend.creditUnratedCalls)).toContain('1 of 2 Copilot requests') + }) + // #1199: only CLI session-store rows carry total_nano_aiu, so a typical user // has hundreds of unrated requests and a confident, wrong credit total. it('estimates credits from token cost for requests with no exact figure', () => { diff --git a/tests/providers/copilot.test.ts b/tests/providers/copilot.test.ts index 19b2f0b60..a4c431b34 100644 --- a/tests/providers/copilot.test.ts +++ b/tests/providers/copilot.test.ts @@ -8,7 +8,7 @@ import { fileURLToPath } from 'node:url' import { ChatCompletionContentPartKind, copilot, createCopilotProvider, findJetBrainsModelToken, getVSCodeGlobalStorageDirs, getVSCodeWorkspaceStorageDirs } from '../../src/providers/copilot.js' import { isSqliteAvailable, isSqliteBusyError } from '../../src/sqlite.js' -import { calculateCost } from '../../src/models.js' +import { calculateCost, fallbackRawModelDisplayName } from '../../src/models.js' import type { ParsedProviderCall } from '../../src/providers/types.js' let tmpDir: string @@ -1125,14 +1125,75 @@ describe('copilot provider - chatSessions parsing', () => { expect(calls.map(c => [c.inputTokens, c.outputTokens])).toEqual([[1000, 20]]) }) - it('keeps metadata precedence without summing duplicate token fields', async () => { + it('takes summed root output over last-round metadata without summing duplicate fields', async () => { const path = join(tmpDir, 'precedence.jsonl') await createChatSessionFile(path, [{ kind: 0, v: { sessionId: 'precedence', requests: [{ requestId: 'r', promptTokens: 1000, completionTokens: 490, modelId: 'copilot/gpt-4o', result: { metadata: { promptTokens: 500, outputTokens: 60 } }, }] } }]) const calls = await collectCalls({ path, project: 'p', provider: 'copilot', sourceType: 'chatsession' }) - expect(calls.map(c => [c.inputTokens, c.outputTokens])).toEqual([[500, 60]]) + expect(calls.map(c => [c.inputTokens, c.outputTokens])).toEqual([[500, 490]]) + }) + + it('keeps metadata-only requests from older VS Code', async () => { + const path = join(tmpDir, 'metadata-only.jsonl') + await createChatSessionFile(path, [{ kind: 0, v: { sessionId: 'metadata-only', requests: [{ + requestId: 'r', modelId: 'copilot/gpt-4o', + result: { metadata: { promptTokens: 500, outputTokens: 60 } }, + }] } }]) + const [call] = await collectCalls({ path, project: 'p', provider: 'copilot', sourceType: 'chatsession' }) + expect([call!.inputTokens, call!.outputTokens, call!.cacheReadInputTokens]).toEqual([500, 60, 0]) + expect(call!.costUSD).toBe(calculateCost('gpt-4o', 500, 60, 0, 0, 0)) + expect(call!.nanoAiu).toBeUndefined() + expect(call!.costFromBilling).toBeUndefined() + }) + + // Copilot Chat 0.69: metadata holds the last of an agent request's model + // calls; completionTokens, modelTotals and copilotCredits cover all of them. + function agentRequest(overrides: Record = {}) { + return chatSessionSampleRequest({ + requestId: 'agent', + promptTokens: 30_100, + completionTokens: 4_300, + modelTotals: [{ model: 'claude-sonnet-4.6', inputTokens: 250_000, cachedTokens: 220_000, outputTokens: 4_300 }], + copilotCredits: 96.5, + sessionCopilotCredits: 140.25, + result: { metadata: { promptTokens: 30_100, outputTokens: 120, resolvedModel: 'claude-sonnet-4-6', summaries: [{ toolCallRoundId: 'round-3', text: 'summary' }] } }, + ...overrides, + }) + } + + it.each(['journal', 'json'])('counts every round of a multi-round agent request (%s)', async format => { + const session = { version: 3, creationDate: 1780157113020, sessionId: 'agent-session', requests: [agentRequest()] } + const path = join(tmpDir, `agent.${format === 'json' ? 'json' : 'jsonl'}`) + if (format === 'json') await writeFile(path, JSON.stringify(session)) + else await createChatSessionFile(path, [{ kind: 0, v: session }]) + const [call, ...rest] = await collectCalls({ path, project: 'p', provider: 'copilot', sourceType: 'chatsession' }) + expect(rest).toEqual([]) + expect([call!.inputTokens, call!.cacheReadInputTokens, call!.cacheCreationInputTokens, call!.outputTokens]).toEqual([30_000, 220_000, 0, 4_300]) + expect(call!.costUSD).toBeCloseTo(0.965, 12) + expect(call!.costFromBilling).toBe(true) + expect(call!.nanoAiu).toBe(96_500_000_000) + expect(call!.costIsEstimated).toBeFalsy() + }) + + it('sums modelTotals across models and falls back to root completionTokens without them', async () => { + const path = join(tmpDir, 'agent-variants.jsonl') + await createChatSessionFile(path, [{ kind: 0, v: { sessionId: 'agent-variants', requests: [ + agentRequest({ requestId: 'two-models', modelTotals: [ + { model: 'claude-sonnet-4.6', inputTokens: 1_000, cachedTokens: 600, outputTokens: 50 }, + { model: 'gpt-5-mini', inputTokens: 500, cachedTokens: 0, outputTokens: 25 }, + ] }), + agentRequest({ requestId: 'no-totals', modelTotals: undefined }), + agentRequest({ requestId: 'credits-only', promptTokens: 0, completionTokens: 0, modelTotals: undefined, copilotCredits: 3.25, result: { metadata: {} } }), + ] } }]) + const calls = await collectCalls({ path, project: 'p', provider: 'copilot', sourceType: 'chatsession' }) + expect(calls.map(c => [c.inputTokens, c.cacheReadInputTokens, c.outputTokens, c.nanoAiu])).toEqual([ + [900, 600, 75, 96_500_000_000], + [30_100, 0, 4_300, 96_500_000_000], + [0, 0, 0, 3_250_000_000], + ]) + expect(calls.map(c => c.costUSD)).toEqual([0.965, 0.965, 0.0325].map(usd => expect.closeTo(usd, 12))) }) it.each([0, -1, '1000', null])('skips invalid or zero root usage without estimating text (%s)', async promptTokens => { @@ -1154,9 +1215,9 @@ describe('copilot provider - chatSessions parsing', () => { expect(calls).toHaveLength(1) expect(calls[0]!.inputTokens).toBe(32543) - expect(calls[0]!.outputTokens).toBe(60) + expect(calls[0]!.outputTokens).toBe(490) expect(calls[0]!.model).toBe('claude-sonnet-4-6') - expect(calls[0]!.costUSD).toBeGreaterThan(0) + expect(calls[0]!.costUSD).toBe(calculateCost('claude-sonnet-4-6', 32543, 490, 0, 0, 0)) }) it('extracts structured skill calls without inferring from prompt text', async () => { @@ -1323,7 +1384,7 @@ describe('copilot provider - chatSessions parsing', () => { await createChatSessionFile(filePath, [ { kind: 0, v: { version: 3, creationDate: 1780157113020, sessionId: 'chat-edit', requests: [] } }, { kind: 2, k: ['requests'], v: [chatSessionSampleRequest()] }, - { kind: 1, k: ['requests', 0, 'result', 'metadata', 'outputTokens'], v: 88 }, + { kind: 1, k: ['requests', 0, 'completionTokens'], v: 88 }, ]) const calls = await collectCalls({ path: filePath, project: 'myproject', provider: 'copilot', sourceType: 'chatsession' }) @@ -1332,6 +1393,20 @@ describe('copilot provider - chatSessions parsing', () => { expect(calls[0]!.outputTokens).toBe(88) }) + it('applies a journal edit inside the nested result metadata', async () => { + const filePath = join(tmpDir, 'append-edit-metadata.jsonl') + await createChatSessionFile(filePath, [ + { kind: 0, v: { version: 3, creationDate: 1780157113020, sessionId: 'chat-edit-meta', requests: [] } }, + { kind: 2, k: ['requests'], v: [chatSessionSampleRequest({ completionTokens: undefined })] }, + { kind: 1, k: ['requests', 0, 'result', 'metadata', 'outputTokens'], v: 88 }, + { kind: 1, k: ['requests', 0, 'result', 'metadata', 'promptTokens'], v: 777 }, + ]) + + const calls = await collectCalls({ path: filePath, project: 'myproject', provider: 'copilot', sourceType: 'chatsession' }) + + expect(calls.map(c => [c.inputTokens, c.outputTokens])).toEqual([[777, 88]]) + }) + it('deduplicates by requestId across parser runs', async () => { const filePath = join(tmpDir, 'dedupe.jsonl') await createChatSessionFile(filePath, [ @@ -1529,6 +1604,13 @@ describe('copilot provider - metadata', () => { expect(copilot.modelDisplayName('gpt-5-mini-2026-01-01')).toBe('GPT-5 Mini') expect(copilot.modelDisplayName('gpt-4.1-mini-2026-01-01')).toBe('GPT-4.1 Mini') }) + + it('does not let a major-version key swallow a dash minor version', () => { + const shown = (id: string) => fallbackRawModelDisplayName(copilot.modelDisplayName(id), id) + expect(shown('claude-sonnet-4-6')).toBe('Sonnet 4.6') + expect(shown('claude-sonnet-4-5')).toBe('Sonnet 4.5') + expect(shown('claude-sonnet-4-20250514')).toBe('Sonnet 4') + }) }) // ---------------------------------------------------------------------------