From 233bd475eae04c77c2e0d13ff1b48a855d161343 Mon Sep 17 00:00:00 2001 From: Uttam Deb <105539415+uttamdeb@users.noreply.github.com> Date: Mon, 5 Oct 2026 10:48:07 +0600 Subject: [PATCH 1/8] Test native Codex tool event attribution --- tests/providers/codex-resume.test.ts | 44 ++++++++++ tests/providers/codex.test.ts | 117 +++++++++++++++++++++++++++ 2 files changed, 161 insertions(+) diff --git a/tests/providers/codex-resume.test.ts b/tests/providers/codex-resume.test.ts index 66878ff80..92d23a7b5 100644 --- a/tests/providers/codex-resume.test.ts +++ b/tests/providers/codex-resume.test.ts @@ -128,6 +128,50 @@ async function parse(cacheDir: string, codexDir = tmpDir): Promise { + it('preserves native-tool identity and last-turn attachment across an appended tail', async () => { + const warmCache = join(tmpDir, 'cache-warm-native-tools') + const coldCache = join(tmpDir, 'cache-cold-native-tools') + const started = (timestamp: string) => JSON.stringify({ + type: 'event_msg', timestamp, payload: { type: 'task_started' }, + }) + const record = (timestamp: string, responseId: string, input: number, output: number) => JSON.stringify({ + type: 'token_usage_record', timestamp, + payload: { response_id: responseId, usage: { input_tokens: input, output_tokens: output } }, + }) + const responseSearch = JSON.stringify({ + type: 'response_item', timestamp: '2026-04-14T10:00:03Z', + payload: { type: 'web_search_call', id: 'ws-resume', status: 'completed', action: { type: 'search', query: 'fixture' } }, + }) + + sessionPath = await writeRollout([ + meta(), + started('2026-04-14T10:00:00Z'), + record('2026-04-14T10:00:02Z', 'resp-native-1', 100, 20), + responseSearch, + started('2026-04-14T10:01:00Z'), + ]) + const first = await parse(warmCache) + expect(first).toHaveLength(1) + expect(first[0]?.tools).toEqual(['WebSearch']) + + await appendFile(sessionPath, [ + JSON.stringify({ + type: 'event_msg', timestamp: '2026-04-14T10:01:01Z', + payload: { type: 'item_completed', item: { type: 'WebSearch', id: 'ws-resume', results: [] } }, + }), + record('2026-04-14T10:01:02Z', 'resp-native-2', 60, 10), + ].join('\n') + '\n') + + readLineCalls.length = 0 + const resumed = await parse(warmCache) + const resumeReads = readLineCalls.filter(c => c.filePath === sessionPath) + expect(resumeReads.length).toBeGreaterThan(0) + expect(resumeReads.every(c => (c.startByteOffset ?? 0) > 0)).toBe(true) + expect(resumed).toHaveLength(2) + expect(resumed.map(call => call.tools)).toEqual([['WebSearch'], []]) + expect(JSON.stringify(resumed)).toBe(JSON.stringify(await parse(coldCache))) + }) + it('keeps the usage-record handover when an appended tail resumes at a task boundary', async () => { const warmCache = join(tmpDir, 'cache-warm') const coldCache = join(tmpDir, 'cache-cold') diff --git a/tests/providers/codex.test.ts b/tests/providers/codex.test.ts index 0a98cfb33..32113cd9b 100644 --- a/tests/providers/codex.test.ts +++ b/tests/providers/codex.test.ts @@ -1521,6 +1521,123 @@ describe('codex provider - fork replay masking against the parent rollout', () = expect((await parseFile(childPath)).map(x => x.inputTokens)).toEqual([500]) }) }) +describe('codex provider - native tool events', () => { + function nativeResponseItem(type: string, item: Record, timestamp: string) { + return JSON.stringify({ type: 'response_item', timestamp, payload: { type, ...item } }) + } + + function completedWebSearch(id: string, timestamp: string) { + return JSON.stringify({ + type: 'event_msg', timestamp, + payload: { + type: 'item_completed', + item: { type: 'WebSearch', id, query: 'fixture query', results: [] }, + }, + }) + } + + async function parseFile(filePath: string): Promise { + const provider = createCodexProvider(tmpDir) + const calls: ParsedProviderCall[] = [] + for await (const call of provider.createSessionParser({ path: filePath, project: 'test', provider: 'codex' }, new Set()).parse()) { + calls.push(call) + } + return calls + } + + it('attributes built-in response tools without changing recorded usage or cost', async () => { + const nativePath = await writeSession(tmpDir, '2026-04-14', 'rollout-native-tools.jsonl', [ + sessionMeta({ session_id: 'sess-native-tools' }), + userMessage('use the built-in tools'), + nativeResponseItem('web_search_call', { id: 'ws-1', status: 'completed', action: { type: 'search', query: 'fixture' } }, '2026-04-14T10:00:10Z'), + nativeResponseItem('tool_search_call', { id: 'ts-1', call_id: 'search-1', status: 'completed', execution: 'server', arguments: { query: 'fixture' } }, '2026-04-14T10:00:11Z'), + nativeResponseItem('image_generation_call', { id: 'ig-1', status: 'completed', result: 'fixture-image-result' }, '2026-04-14T10:00:12Z'), + tokenUsageRecord({ responseId: 'resp-native-tools', usage: { input: 1000, cached: 200, output: 100 } }), + ]) + const controlPath = await writeSession(tmpDir, '2026-04-14', 'rollout-native-control.jsonl', [ + sessionMeta({ session_id: 'sess-native-control' }), + tokenUsageRecord({ responseId: 'resp-native-control', usage: { input: 1000, cached: 200, output: 100 } }), + ]) + + const [native] = await parseFile(nativePath) + const [control] = await parseFile(controlPath) + + expect(native).toBeDefined() + expect(native!.tools).toEqual(['WebSearch', 'ToolSearch', 'ImageGeneration']) + expect(native!.toolSequence?.map(step => step[0]?.tool)).toEqual(native!.tools) + expect(native).toMatchObject({ inputTokens: 800, cachedInputTokens: 200, outputTokens: 100, webSearchRequests: 0 }) + expect(native!.costUSD).toBe(control!.costUSD) + }) + + it('deduplicates partial, completed, and mirrored WebSearch events by item identity', async () => { + const filePath = await writeSession(tmpDir, '2026-04-14', 'rollout-native-dedup.jsonl', [ + sessionMeta({ session_id: 'sess-native-dedup' }), + nativeResponseItem('web_search_call', { id: 'ws-shared', status: 'searching', action: { type: 'search', query: 'fixture' } }, '2026-04-14T10:00:10Z'), + nativeResponseItem('web_search_call', { id: 'ws-shared', status: 'completed', action: { type: 'search', query: 'fixture' } }, '2026-04-14T10:00:11Z'), + completedWebSearch('ws-shared', '2026-04-14T10:00:12Z'), + completedWebSearch('ws-item-only', '2026-04-14T10:00:13Z'), + tokenUsageRecord({ responseId: 'resp-native-dedup', usage: { input: 100, output: 20 } }), + ]) + + const [call] = await parseFile(filePath) + + expect(call?.tools).toEqual(['WebSearch', 'WebSearch']) + expect(call?.toolSequence?.map(step => step[0]?.tool)).toEqual(['WebSearch', 'WebSearch']) + }) + + it('attaches native tool events after the final usage record to that turn', async () => { + const filePath = await writeSession(tmpDir, '2026-04-14', 'rollout-native-trailing.jsonl', [ + sessionMeta({ session_id: 'sess-native-trailing' }), + tokenUsageRecord({ responseId: 'resp-native-trailing', usage: { input: 100, output: 20 } }), + completedWebSearch('ws-trailing', '2026-04-14T10:01:00Z'), + ]) + + const calls = await parseFile(filePath) + + expect(calls).toHaveLength(1) + expect(calls[0]?.tools).toEqual(['WebSearch']) + }) + + it('suppresses native tool events replayed by a fork while keeping divergent work', async () => { + const parentPath = await writeSession(tmpDir, '2026-04-14', 'rollout-native-parent.jsonl', [ + sessionMeta({ session_id: 'sess-native-parent' }), + nativeResponseItem('web_search_call', { id: 'ws-parent', status: 'completed' }, '2026-04-14T10:00:01Z'), + tokenUsageRecord({ timestamp: '2026-04-14T10:00:02Z', responseId: 'resp-parent', usage: { input: 100, output: 20 } }), + ]) + const forkPath = await writeSession(tmpDir, '2026-04-14', 'rollout-native-fork.jsonl', [ + sessionMeta({ session_id: 'sess-native-fork', forked_from_id: 'sess-native-parent', timestamp: '2026-04-14T10:00:10Z' }), + nativeResponseItem('web_search_call', { id: 'ws-parent', status: 'completed' }, '2026-04-14T10:00:10.100Z'), + tokenUsageRecord({ timestamp: '2026-04-14T10:00:10.200Z', responseId: 'resp-parent', usage: { input: 100, output: 20 } }), + completedWebSearch('ws-parent', '2026-04-14T10:00:10.300Z'), + nativeResponseItem('web_search_call', { id: 'ws-fork', status: 'completed' }, '2026-04-14T10:00:14Z'), + tokenUsageRecord({ timestamp: '2026-04-14T10:00:14.100Z', responseId: 'resp-fork', usage: { input: 80, output: 15 } }), + ]) + + const parentCalls = await parseFile(parentPath) + const forkCalls = await parseFile(forkPath) + + expect(parentCalls[0]?.tools).toEqual(['WebSearch']) + expect(forkCalls).toHaveLength(1) + expect(forkCalls[0]?.tools).toEqual(['WebSearch']) + }) + + it('attributes a large image-generation result through the compact line decoder', async () => { + const largeImage = JSON.stringify({ + type: 'response_item', timestamp: '2026-04-14T10:00:10Z', + payload: { type: 'image_generation_call', id: 'ig-large', status: 'completed', revised_prompt: 'fixture', result: 'a'.repeat(2 * 1024 * 1024) }, + }) + const filePath = await writeSession(tmpDir, '2026-04-14', 'rollout-native-large-image.jsonl', [ + sessionMeta({ session_id: 'sess-native-large-image' }), + largeImage, + tokenUsageRecord({ responseId: 'resp-native-large-image', usage: { input: 100, output: 20 } }), + ]) + + const calls = await parseFile(filePath) + + expect(calls).toHaveLength(1) + expect(calls[0]?.tools).toEqual(['ImageGeneration']) + }) +}) describe('codex provider - token_usage_record accounting', () => { async function parseCalls(lines: string[]): Promise { From fbe320b37f1b0a963892f2c1368360a01fa5090c Mon Sep 17 00:00:00 2001 From: Uttam Deb <105539415+uttamdeb@users.noreply.github.com> Date: Mon, 5 Oct 2026 10:53:15 +0600 Subject: [PATCH 2/8] Parse native Codex tool events --- src/providers/codex.ts | 142 ++++++++++++++++++++++++++++++++++++++++- 1 file changed, 139 insertions(+), 3 deletions(-) diff --git a/src/providers/codex.ts b/src/providers/codex.ts index d56130135..07732f4b9 100644 --- a/src/providers/codex.ts +++ b/src/providers/codex.ts @@ -130,6 +130,7 @@ type CodexEntry = { timestamp?: string payload?: { type?: string + id?: string turn_id?: string call_id?: string started_at?: number @@ -157,6 +158,7 @@ type CodexEntry = { /// Raw shape of the `thread_settings_applied` payload; only the service /// tier is read off it. thread_settings?: { service_tier?: string } + item?: Record invocation?: { server?: string; tool?: string } content?: Array<{ type?: string; text?: string }> info?: { @@ -169,6 +171,44 @@ type CodexEntry = { } } +type CodexNativeToolEvent = { + tool: 'WebSearch' | 'ToolSearch' | 'ImageGeneration' + identity?: string +} + +type CodexPendingNativeToolEvent = { + tool: CodexNativeToolEvent['tool'] + toolIndex: number + sequenceIndex: number +} + +function nativeToolEvent(entry: CodexEntry): CodexNativeToolEvent | null { + const payload = entry.payload + if (!payload) return null + + let tool: CodexNativeToolEvent['tool'] | null = null + let identity: string | undefined + if (entry.type === 'response_item') { + if (payload.type === 'web_search_call') tool = 'WebSearch' + else if (payload.type === 'tool_search_call') tool = 'ToolSearch' + else if (payload.type === 'image_generation_call') tool = 'ImageGeneration' + if (!tool) return null + identity = typeof payload.id === 'string' && payload.id + ? payload.id + : typeof payload.call_id === 'string' && payload.call_id + ? payload.call_id + : undefined + } else if (entry.type === 'event_msg' && payload.type === 'item_completed') { + const item = payload.item + if (!item || item['type'] !== 'WebSearch') return null + tool = 'WebSearch' + identity = typeof item['id'] === 'string' && item['id'] ? item['id'] : undefined + } + + if (!tool) return null + return { tool, ...(identity ? { identity: `${tool}:${identity}` } : {}) } +} + type CodexTokenUsage = { input_tokens?: number cached_input_tokens?: number @@ -520,6 +560,14 @@ function parseCodexLine(line: string | Buffer): CodexEntry | null { const compactModelName = getRawJsonStringField(pHead, 'model_name') const compactLastUsage = getRawTokenUsage(pHead, 'last_token_usage') const compactTotalUsage = getRawTokenUsage(pHead, 'total_token_usage') + const itemWindow = type === 'event_msg' && payloadType === 'item_completed' + ? getRawPayloadFieldWindow(line, 'item') + : undefined + const compactItemType = itemWindow ? getRawJsonStringField(itemWindow, 'type') : undefined + const compactItemId = itemWindow ? getRawJsonStringField(itemWindow, 'id') : undefined + const compactItem = compactItemType + ? { type: compactItemType, ...(compactItemId ? { id: compactItemId } : {}) } + : undefined const compactInfo = compactModel || compactModelName || compactLastUsage || compactTotalUsage ? { model: compactModel, model_name: compactModelName, last_token_usage: compactLastUsage, total_token_usage: compactTotalUsage } : undefined @@ -530,6 +578,7 @@ function parseCodexLine(line: string | Buffer): CodexEntry | null { timestamp: getRawJsonStringField(head, 'timestamp'), payload: { type: payloadType, + id: getRawJsonStringField(pHead, 'id'), role, cwd: payloadString('cwd'), model_provider: payloadString('model_provider'), @@ -541,6 +590,7 @@ function parseCodexLine(line: string | Buffer): CodexEntry | null { : undefined, model: compactModel, name: payloadString('name'), + item: compactItem, invocation, call_id: getRawJsonStringField(pHead, 'call_id'), turn_id: getRawJsonStringField(pHead, 'turn_id'), @@ -716,6 +766,10 @@ type CodexResumeState = { prevReasoning: number pendingTools: string[] pendingToolSequence: ToolCall[][] + /// Native tool identities make response_item / item_completed mirrors one + /// attribution after an append resume or a task-boundary checkpoint. + pendingNativeToolEvents?: CodexPendingNativeToolEvent[] + nativeToolEventIdentities?: string[] /// Optional so a resume state written before skills attribution existed still /// decodes (isResumeState does not require it); absent reads as "no skills". pendingSkills?: string[] @@ -741,6 +795,32 @@ type CodexResumeState = { function isResumeState(value: unknown): value is CodexResumeState { if (!value || typeof value !== 'object') return false const v = value as Record + const pendingTools = v['pendingTools'] + const pendingToolSequence = v['pendingToolSequence'] + const pendingNativeToolEvents = v['pendingNativeToolEvents'] + const nativeToolEventIdentities = v['nativeToolEventIdentities'] + const validPendingNativeToolEvents = pendingNativeToolEvents === undefined || ( + Array.isArray(pendingNativeToolEvents) + && Array.isArray(pendingTools) + && Array.isArray(pendingToolSequence) + && pendingNativeToolEvents.every(event => { + if (!event || typeof event !== 'object') return false + const item = event as Record + const toolIndex = item['toolIndex'] + const sequenceIndex = item['sequenceIndex'] + return (item['tool'] === 'WebSearch' || item['tool'] === 'ToolSearch' || item['tool'] === 'ImageGeneration') + && typeof toolIndex === 'number' + && Number.isInteger(toolIndex) + && toolIndex >= 0 + && toolIndex < pendingTools.length + && pendingTools[toolIndex] === item['tool'] + && typeof sequenceIndex === 'number' + && Number.isInteger(sequenceIndex) + && sequenceIndex >= 0 + && sequenceIndex < pendingToolSequence.length + && Array.isArray(pendingToolSequence[sequenceIndex]) + }) + ) return typeof v['sessionId'] === 'string' && typeof v['forkedFromId'] === 'string' && (v['forkReplayState'] === undefined || isCodexForkReplayState(v['forkReplayState'])) @@ -753,8 +833,10 @@ function isResumeState(value: unknown): value is CodexResumeState { && typeof v['prevCacheWrite'] === 'number' && typeof v['prevOutput'] === 'number' && typeof v['prevReasoning'] === 'number' - && Array.isArray(v['pendingTools']) - && Array.isArray(v['pendingToolSequence']) + && Array.isArray(pendingTools) + && Array.isArray(pendingToolSequence) + && validPendingNativeToolEvents + && (nativeToolEventIdentities === undefined || (Array.isArray(nativeToolEventIdentities) && nativeToolEventIdentities.every(id => typeof id === 'string'))) && typeof v['pendingUserMessage'] === 'string' && typeof v['pendingOutputChars'] === 'number' && typeof v['pendingLocAdded'] === 'number' @@ -839,6 +921,10 @@ function createParser(source: SessionSource, seenKeys: Set, capture?: { let prevReasoning = resume?.state.prevReasoning ?? 0 let pendingTools: string[] = resume ? [...resume.state.pendingTools] : [] let pendingToolSequence: ToolCall[][] = resume ? [...resume.state.pendingToolSequence] : [] + let pendingNativeToolEvents: CodexPendingNativeToolEvent[] = resume?.state.pendingNativeToolEvents + ? resume.state.pendingNativeToolEvents.map(event => ({ ...event })) + : [] + const nativeToolEventIdentities = new Set(resume?.state.nativeToolEventIdentities ?? []) let pendingSkills: string[] = resume?.state.pendingSkills ? [...resume.state.pendingSkills] : [] // Attribution names already emitted from a `response_item` tool call. // Codex's item model repeats a finished shell command as @@ -879,6 +965,16 @@ function createParser(source: SessionSource, seenKeys: Set, capture?: { pendingToolSequence.push([{ tool: 'Skill', file: skill }]) } } + + const attributeNativeToolEvent = (event: CodexNativeToolEvent): void => { + if (event.identity && nativeToolEventIdentities.has(event.identity)) return + if (event.identity) nativeToolEventIdentities.add(event.identity) + const toolIndex = pendingTools.length + const sequenceIndex = pendingToolSequence.length + pendingTools.push(event.tool) + pendingToolSequence.push([{ tool: event.tool }]) + pendingNativeToolEvents.push({ tool: event.tool, toolIndex, sequenceIndex }) + } let pendingUserMessage = resume?.state.pendingUserMessage ?? '' let pendingOutputChars = resume?.state.pendingOutputChars ?? 0 // Rich-session-capture: edit LOC deltas and failed-patch count accumulated @@ -912,6 +1008,28 @@ function createParser(source: SessionSource, seenKeys: Set, capture?: { let taskActiveStartedAt: number | undefined = resume?.state.taskActiveStartedAt const openToolStarts = new Map() + // Tool items can be serialized after their response's usage record. Keep + // normal forward attribution as the first choice, but attach any still + // pending native items to the last call in this task before it is flushed. + // The parallel indexes let this move just the native items without + // disturbing other pending tool attribution across a resume boundary. + const attachPendingNativeToolEvents = (target: ParsedProviderCall | undefined): void => { + if (!target || pendingNativeToolEvents.length === 0) return + const ordered = [...pendingNativeToolEvents].sort((a, b) => a.sequenceIndex - b.sequenceIndex) + target.tools.push(...ordered.map(event => event.tool)) + target.toolSequence = [ + ...(target.toolSequence ?? []), + ...ordered.map(event => [{ tool: event.tool }]), + ] + for (const event of [...pendingNativeToolEvents].sort((a, b) => b.toolIndex - a.toolIndex)) { + pendingTools.splice(event.toolIndex, 1) + } + for (const event of [...pendingNativeToolEvents].sort((a, b) => b.sequenceIndex - a.sequenceIndex)) { + pendingToolSequence.splice(event.sequenceIndex, 1) + } + pendingNativeToolEvents = [] + } + const accountUsage = (entry: CodexEntry, usage: CodexTokenUsage, dedupKey: string): void => { const inputTokens = usage.input_tokens ?? 0 const cachedInputTokens = usage.cached_input_tokens ?? 0 @@ -992,6 +1110,7 @@ function createParser(source: SessionSource, seenKeys: Set, capture?: { pendingTools = [] pendingToolSequence = [] + pendingNativeToolEvents = [] pendingSkills = [] pendingUserMessage = '' pendingOutputChars = 0 @@ -1092,6 +1211,9 @@ function createParser(source: SessionSource, seenKeys: Set, capture?: { entry.payload?.type === 'function_call_output' || entry.payload?.type === 'custom_tool_call' || entry.payload?.type === 'custom_tool_call_output' || + entry.payload?.type === 'web_search_call' || + entry.payload?.type === 'tool_search_call' || + entry.payload?.type === 'image_generation_call' || entry.payload?.type === 'mcp_tool_call_end' || entry.payload?.type === 'item_completed' || entry.payload?.type === 'patch_apply_end' @@ -1129,6 +1251,7 @@ function createParser(source: SessionSource, seenKeys: Set, capture?: { if (entry.type === 'event_msg' && entry.payload?.type === 'task_started') { // Emit the previous task. If it never reached task_complete its timing // fields simply stay unset, matching the un-buffered behaviour. + attachPendingNativeToolEvents(pendingTaskCalls.at(-1)) results.push(...pendingTaskCalls) pendingTaskCalls = [] taskGeneratedTokens = 0 @@ -1158,6 +1281,8 @@ function createParser(source: SessionSource, seenKeys: Set, capture?: { prevReasoning, pendingTools: [...pendingTools], pendingToolSequence: [...pendingToolSequence], + pendingNativeToolEvents: pendingNativeToolEvents.map(event => ({ ...event })), + nativeToolEventIdentities: [...nativeToolEventIdentities], pendingSkills: [...pendingSkills], pendingUserMessage, pendingOutputChars, @@ -1173,6 +1298,15 @@ function createParser(source: SessionSource, seenKeys: Set, capture?: { continue } + const nativeEvent = nativeToolEvent(entry) + if (nativeEvent) { + // Status transitions share an item identity. The first observed + // record proves tool activity even when the rollout ends before its + // completed status; mirrors from the item model are ignored by id. + attributeNativeToolEvent(nativeEvent) + continue + } + if (entry.type === 'response_item' && (entry.payload?.type === 'function_call' || entry.payload?.type === 'custom_tool_call')) { const rawName = entry.payload.name ?? '' const mapped = toolNameMap[rawName] ?? rawName @@ -1368,7 +1502,7 @@ function createParser(source: SessionSource, seenKeys: Set, capture?: { const timestamp = entry.timestamp ?? '' const dedupKey = `codex:${sessionId}:${timestamp}:est${estCounter++}` - if (seenKeys.has(dedupKey)) { pendingTools = []; pendingToolSequence = []; pendingSkills = []; pendingUserMessage = ''; pendingOutputChars = 0; pendingLocAdded = 0; pendingLocRemoved = 0; pendingEditFailed = 0; continue } + if (seenKeys.has(dedupKey)) { pendingTools = []; pendingToolSequence = []; pendingNativeToolEvents = []; pendingSkills = []; pendingUserMessage = ''; pendingOutputChars = 0; pendingLocAdded = 0; pendingLocRemoved = 0; pendingEditFailed = 0; continue } seenKeys.add(dedupKey) // An estimated prompt can cross a long-context threshold and tier @@ -1408,6 +1542,7 @@ function createParser(source: SessionSource, seenKeys: Set, capture?: { pendingTools = [] pendingToolSequence = [] + pendingNativeToolEvents = [] pendingSkills = [] pendingUserMessage = '' pendingOutputChars = 0 @@ -1525,6 +1660,7 @@ function createParser(source: SessionSource, seenKeys: Set, capture?: { } // Flush the final task, which has no following task_started to trigger it. + attachPendingNativeToolEvents(pendingTaskCalls.at(-1)) results.push(...pendingTaskCalls) const resumeWrite = resumeState ? { offset: resumeOffset, state: resumeState, callCount: resumeCallCount } : undefined From 5343d6080c0e6f2be494975fa68136d1d91683c5 Mon Sep 17 00:00:00 2001 From: Uttam Deb <105539415+uttamdeb@users.noreply.github.com> Date: Mon, 5 Oct 2026 11:13:33 +0600 Subject: [PATCH 3/8] Refresh cached Codex tool attribution --- docs/providers/codex.md | 18 ++++ src/codex-cache.ts | 4 +- src/daily-cache.ts | 5 +- src/providers/codex.ts | 80 ++++++++++++---- src/session-cache.ts | 5 +- tests/codex-cache-invalidation.test.ts | 56 +++++++++++ .../daily-cache-version-rederivation.test.ts | 93 +++++++++++++++++++ tests/providers/codex.test.ts | 76 ++++++++++++++- 8 files changed, 312 insertions(+), 25 deletions(-) diff --git a/docs/providers/codex.md b/docs/providers/codex.md index a47de5762..2d69cc2c2 100644 --- a/docs/providers/codex.md +++ b/docs/providers/codex.md @@ -71,6 +71,24 @@ Four layers, in order: Estimated events that fall back to char-counting use `codex:::est`. +## Native tool attribution + +`web_search_call`, `tool_search_call`, and `image_generation_call` are recorded +as `WebSearch`, `ToolSearch`, and `ImageGeneration`. `event_msg/item_completed` +records whose item type is `WebSearch` share the same `WebSearch` identity. +Repeated status updates and mirrored records collapse by their stable item id +(or `call_id` for tool search); records without one remain distinct because +query or result text is not a safe identity. + +The first observed status records activity, whether partial or completed. Native +items add tool attribution only: they do not change token usage, +`webSearchRequests`, or cost. Fork replay suppression applies to response-item +built-ins as well as item-completed records. When a native tool item arrives +after the final usage record in its task, it is attached to that task's last +call. Large image results and search result arrays stay out of the compact +decoded entry; the decoder scans direct id/type fields and copies only bounded +field windows. + ## Quirks - Newer Codex builds emit one `token_usage_record` per model response with a `response_id`. Those records are the preferred usage source from the first usable record onward; earlier `token_count` events in a mixed rollout still count, and malformed records leave the legacy fallback active. diff --git a/src/codex-cache.ts b/src/codex-cache.ts index 967af2154..aeda8c530 100644 --- a/src/codex-cache.ts +++ b/src/codex-cache.ts @@ -68,7 +68,9 @@ import { isWslUncPath } from './wsl.js' // GPT-5.3 Codex Spark instead of $0. Stored costUSD and speed are the old ones. // v22: a fork's replay burst drops only records found in the parent rollout // and counts the rest. v21 entries dropped the whole burst. -export const CODEX_CACHE_VERSION = 22 +// v23: attribute the Responses built-in tools and item-model WebSearch records; +// v22 exact entries have no such tool activity, so they must reparse. +export const CODEX_CACHE_VERSION = 23 export const CODEX_LEGACY_CACHE_FILE = 'codex-results.json' export function codexCacheFileName(version = CODEX_CACHE_VERSION): string { return `codex-results.v${version}.json` diff --git a/src/daily-cache.ts b/src/daily-cache.ts index 3ded9cc7b..dd4bcba08 100644 --- a/src/daily-cache.ts +++ b/src/daily-cache.ts @@ -361,7 +361,10 @@ import type { DateRange, ProjectSummary } from './types.js' // copilotCredits, and a chat session that also ran on the agent host is left // to it. That twin drops calls, so copilot joins // PENDING_REDERIVE_PROVIDER_VERSIONS at 75. -export const DAILY_CACHE_VERSION = 75 +// v76: native Codex tool events now inform turn classification. Re-derive +// settled days so their category totals reflect the recovered tool activity; +// calls, tokens, and cost are unchanged. +export const DAILY_CACHE_VERSION = 76 const MIN_SUPPORTED_VERSION = 28 /// Providers whose per-day CALL COUNT means something different at diff --git a/src/providers/codex.ts b/src/providers/codex.ts index 07732f4b9..ceb5aed60 100644 --- a/src/providers/codex.ts +++ b/src/providers/codex.ts @@ -320,17 +320,14 @@ function getRawJsonNumberField(head: string, field: string): number | undefined return Number.isFinite(value) ? value : undefined } -function getRawPayloadFieldWindow(source: Buffer, field: string, windowBytes = 4096): string | undefined { - const payloadKey = Buffer.from('"payload"') - const payloadIndex = source.indexOf(payloadKey) - if (payloadIndex < 0) return undefined - let payloadStart = source.indexOf(0x7b, payloadIndex + payloadKey.length) // { - if (payloadStart < 0) return undefined +type RawJsonFieldBounds = { keyStart: number; valueStart: number } +function findRawObjectField(source: Buffer, objectStart: number, field: string): RawJsonFieldBounds | undefined { + const fieldBytes = Buffer.from(field, 'utf-8') let depth = 0 let inString = false let escaped = false - for (let i = payloadStart; i < source.length; i++) { + for (let i = objectStart; i < source.length; i++) { const byte = source[i]! if (inString) { if (escaped) escaped = false @@ -349,11 +346,16 @@ function getRawPayloadFieldWindow(source: Buffer, field: string, windowBytes = 4 if (keyByte === 0x22) break } if (depth === 1 && keyEnd < source.length) { - const key = source.subarray(keyStart, keyEnd).toString('utf-8') - let valueStart = keyEnd + 1 - while (valueStart < source.length && (source[valueStart] === 0x20 || source[valueStart] === 0x09 || source[valueStart] === 0x0a || source[valueStart] === 0x0d)) valueStart++ - if (source[valueStart] === 0x3a && key === field) { - return source.subarray(i, Math.min(source.length, i + windowBytes)).toString('utf-8') + // Most strings at depth 1 are values, including potentially huge + // image results. Check for the key/value delimiter and exact short-key + // length before comparing bytes so scanning never materializes them. + let colon = keyEnd + 1 + while (colon < source.length && (source[colon] === 0x20 || source[colon] === 0x09 || source[colon] === 0x0a || source[colon] === 0x0d)) colon++ + if (source[colon] === 0x3a && keyEnd - keyStart === fieldBytes.length && source.subarray(keyStart, keyEnd).equals(fieldBytes)) { + let valueStart = colon + valueStart++ + while (valueStart < source.length && (source[valueStart] === 0x20 || source[valueStart] === 0x09 || source[valueStart] === 0x0a || source[valueStart] === 0x0d)) valueStart++ + return { keyStart: i, valueStart } } } i = keyEnd @@ -363,11 +365,37 @@ function getRawPayloadFieldWindow(source: Buffer, field: string, windowBytes = 4 if (byte === 0x22) inString = true else if (byte === 0x7b || byte === 0x5b) depth++ // { or [ else if (byte === 0x7d || byte === 0x5d) depth-- // } or ] - if (depth < 0) break + if (depth <= 0) break } return undefined } +function getRawObjectFieldWindow(source: Buffer, objectStart: number, field: string, windowBytes = 4096): string | undefined { + const bounds = findRawObjectField(source, objectStart, field) + return bounds ? source.subarray(bounds.keyStart, Math.min(source.length, bounds.keyStart + windowBytes)).toString('utf-8') : undefined +} + +function getRawPayloadObjectStart(source: Buffer): number | undefined { + const payloadKey = Buffer.from('"payload"') + const payloadIndex = source.indexOf(payloadKey) + if (payloadIndex < 0) return undefined + const payloadStart = source.indexOf(0x7b, payloadIndex + payloadKey.length) // { + return payloadStart < 0 ? undefined : payloadStart +} + +function getRawPayloadFieldWindow(source: Buffer, field: string, windowBytes = 4096): string | undefined { + const payloadStart = getRawPayloadObjectStart(source) + return payloadStart === undefined ? undefined : getRawObjectFieldWindow(source, payloadStart, field, windowBytes) +} + +function getRawPayloadItemFieldWindow(source: Buffer, field: string, windowBytes = 4096): string | undefined { + const payloadStart = getRawPayloadObjectStart(source) + if (payloadStart === undefined) return undefined + const item = findRawObjectField(source, payloadStart, 'item') + if (!item || source[item.valueStart] !== 0x7b) return undefined + return getRawObjectFieldWindow(source, item.valueStart, field, windowBytes) +} + function getRawDurationMs(head: string): number | undefined { const objectMatch = /"duration"\s*:\s*\{\s*"secs"\s*:\s*(-?\d+(?:\.\d+)?)\s*,\s*"nanos"\s*:\s*(-?\d+(?:\.\d+)?)\s*\}/.exec(head) if (objectMatch) { @@ -560,14 +588,28 @@ function parseCodexLine(line: string | Buffer): CodexEntry | null { const compactModelName = getRawJsonStringField(pHead, 'model_name') const compactLastUsage = getRawTokenUsage(pHead, 'last_token_usage') const compactTotalUsage = getRawTokenUsage(pHead, 'total_token_usage') - const itemWindow = type === 'event_msg' && payloadType === 'item_completed' - ? getRawPayloadFieldWindow(line, 'item') + const itemTypeWindow = type === 'event_msg' && payloadType === 'item_completed' + ? getRawPayloadItemFieldWindow(line, 'type') : undefined - const compactItemType = itemWindow ? getRawJsonStringField(itemWindow, 'type') : undefined - const compactItemId = itemWindow ? getRawJsonStringField(itemWindow, 'id') : undefined + const itemIdWindow = type === 'event_msg' && payloadType === 'item_completed' + ? getRawPayloadItemFieldWindow(line, 'id') + : undefined + const compactItemType = itemTypeWindow ? getRawJsonStringField(itemTypeWindow, 'type') : undefined + const compactItemId = itemIdWindow ? getRawJsonStringField(itemIdWindow, 'id') : undefined const compactItem = compactItemType ? { type: compactItemType, ...(compactItemId ? { id: compactItemId } : {}) } : undefined + const isNativeResponseTool = type === 'response_item' && ( + payloadType === 'web_search_call' || payloadType === 'tool_search_call' || payloadType === 'image_generation_call' + ) + const nativeIdWindow = isNativeResponseTool ? getRawPayloadFieldWindow(line, 'id') : undefined + const nativeCallIdWindow = isNativeResponseTool ? getRawPayloadFieldWindow(line, 'call_id') : undefined + const compactId = isNativeResponseTool + ? nativeIdWindow ? getRawJsonStringField(nativeIdWindow, 'id') : undefined + : getRawJsonStringField(pHead, 'id') + const compactCallId = isNativeResponseTool + ? nativeCallIdWindow ? getRawJsonStringField(nativeCallIdWindow, 'call_id') : undefined + : getRawJsonStringField(pHead, 'call_id') const compactInfo = compactModel || compactModelName || compactLastUsage || compactTotalUsage ? { model: compactModel, model_name: compactModelName, last_token_usage: compactLastUsage, total_token_usage: compactTotalUsage } : undefined @@ -578,7 +620,7 @@ function parseCodexLine(line: string | Buffer): CodexEntry | null { timestamp: getRawJsonStringField(head, 'timestamp'), payload: { type: payloadType, - id: getRawJsonStringField(pHead, 'id'), + id: compactId, role, cwd: payloadString('cwd'), model_provider: payloadString('model_provider'), @@ -592,7 +634,7 @@ function parseCodexLine(line: string | Buffer): CodexEntry | null { name: payloadString('name'), item: compactItem, invocation, - call_id: getRawJsonStringField(pHead, 'call_id'), + call_id: compactCallId, turn_id: getRawJsonStringField(pHead, 'turn_id'), // #1616: oversized thread_settings_applied records (a fat permission // profile or plugin list can push them past the buffer threshold) still diff --git a/src/session-cache.ts b/src/session-cache.ts index 65c65ba4f..b45491fce 100644 --- a/src/session-cache.ts +++ b/src/session-cache.ts @@ -429,8 +429,11 @@ export const PROVIDER_PARSE_VERSIONS: Record = { // 'standard'), and `gpt-reserve` now splits cache writes like GPT-5.6 Luna. // codex-fork-mask-v1: a fork's replay burst drops only records the parent // rollout holds, so cached forks that dropped the whole burst re-parse. + // native-tool-events-v1: capture built-in Responses tools and item-model + // WebSearch mirrors, preserving the first observed item identity across + // cached turns and append checkpoints. // Compose every suffix so cached sessions receive all accounting fixes. - codex: 'mcp-attribution-v5-est-cost-active-timing-mcp-wait-rich-capture-v1-cross-provider-pr-v1-session-meta-model-v1-session-meta-fields-v1-codex-pricing-v1-codex-tps-v1-codex-mcp-skills-v1-activity-price-v1-fork-replay-burst-v1-codex-token-usage-record-v1-codex-priority-tier-v1-codex-auto-review-date-v1-codex-flex-reserve-v1-codex-fork-mask-v1', + codex: 'mcp-attribution-v5-est-cost-active-timing-mcp-wait-rich-capture-v1-cross-provider-pr-v1-session-meta-model-v1-session-meta-fields-v1-codex-pricing-v1-codex-tps-v1-codex-mcp-skills-v1-activity-price-v1-fork-replay-burst-v1-codex-token-usage-record-v1-codex-priority-tier-v1-codex-auto-review-date-v1-codex-flex-reserve-v1-codex-fork-mask-v1-native-tool-events-v1', // reported-cost-v1: cached Crush calls stored costUSD: undefined and must // re-parse to keep the recorded session cost. crush: 'reported-cost-v1', diff --git a/tests/codex-cache-invalidation.test.ts b/tests/codex-cache-invalidation.test.ts index 0c70f7e31..d68838ff7 100644 --- a/tests/codex-cache-invalidation.test.ts +++ b/tests/codex-cache-invalidation.test.ts @@ -15,6 +15,7 @@ import { join } from 'path' import { clearSessionCache, parseAllSessions } from '../src/parser.js' import { PROVIDER_PARSE_VERSIONS, type SessionCache } from '../src/session-cache.js' import { parseLiteLLMEntry, restorePricingState, snapshotPricingState } from '../src/models.js' +import { clearCodexMemCaches, CODEX_CACHE_VERSION, codexCacheFileName } from '../src/codex-cache.js' import { readCacheOnDisk, writeCacheOnDisk } from './fixtures/session-cache-io.js' const testRoot = vi.hoisted(() => { @@ -235,4 +236,59 @@ describe('codex parser change invalidates stale session-cache (#478/#513)', () = restorePricingState(snap) } }) + it('refreshes warm caches to capture native Codex tool events', async () => { + const sessionDir = join(CODEX_HOME, 'sessions', '2026', '10', '05') + await mkdir(sessionDir, { recursive: true }) + await mkdir(CACHE_DIR, { recursive: true }) + const lines = [ + JSON.stringify({ type: 'session_meta', timestamp: '2026-10-05T10:00:00Z', payload: { session_id: 'sess-native-tool-cache', model: 'gpt-5.5', cwd: '/Users/test/proj', originator: 'codex_cli_rs' } }), + JSON.stringify({ type: 'response_item', timestamp: '2026-10-05T10:00:10Z', payload: { type: 'web_search_call', id: 'ws-cache', status: 'completed', action: { type: 'search', query: 'fixture' } } }), + JSON.stringify({ type: 'event_msg', timestamp: '2026-10-05T10:00:11Z', payload: { type: 'item_completed', item: { type: 'WebSearch', id: 'ws-cache', results: [] } } }), + JSON.stringify({ type: 'token_usage_record', timestamp: '2026-10-05T10:00:12Z', payload: { response_id: 'resp-native-tool-cache', model: 'gpt-5.5', usage: { input_tokens: 1000, output_tokens: 200 } } }), + ] + await writeFile(join(sessionDir, 'rollout-native-tool-cache.jsonl'), lines.join('\n') + '\n') + + clearSessionCache() + clearCodexMemCaches() + const fresh = await parseAllSessions(undefined, 'codex') + const freshSession = fresh.flatMap(project => project.sessions).find(session => session.sessionId === 'sess-native-tool-cache') + expect(freshSession?.toolBreakdown['WebSearch']?.calls).toBe(1) + + // Restore a pre-native-tool session section and pre-fix Codex result file. + // The provider fingerprint forces the session cache to ask the parser, and + // the older result version must not serve its tool-less exact entry. + const sessionCache = await readCacheOnDisk() as SessionCache + sessionCache.providers['codex']!.envFingerprint = preUsageRecordFingerprint() + for (const file of Object.values(sessionCache.providers['codex']!.files)) { + for (const turn of file.turns) { + for (const call of turn.calls) { + call.tools = call.tools.filter(tool => tool !== 'WebSearch') + call.toolSequence = call.toolSequence?.filter(step => step.every(tool => tool.tool !== 'WebSearch')) + } + } + } + await writeCacheOnDisk(sessionCache) + + const rawPath = join(CACHE_DIR, codexCacheFileName()) + const rawCache = JSON.parse(await readFile(rawPath, 'utf8')) as { + version: number + files: Record> }> }> + } + for (const file of Object.values(rawCache.files)) { + for (const call of file.calls ?? []) { + call.tools = (call.tools ?? []).filter(tool => tool !== 'WebSearch') + call.toolSequence = call.toolSequence?.filter(step => step.every(tool => tool.tool !== 'WebSearch')) + } + } + const oldVersion = CODEX_CACHE_VERSION - 1 + rawCache.version = oldVersion + await rm(rawPath, { force: true }) + await writeFile(join(CACHE_DIR, codexCacheFileName(oldVersion)), JSON.stringify(rawCache)) + + clearSessionCache() + clearCodexMemCaches() + const refreshed = await parseAllSessions(undefined, 'codex') + const refreshedSession = refreshed.flatMap(project => project.sessions).find(session => session.sessionId === 'sess-native-tool-cache') + expect(refreshedSession?.toolBreakdown['WebSearch']?.calls).toBe(1) + }) }) diff --git a/tests/daily-cache-version-rederivation.test.ts b/tests/daily-cache-version-rederivation.test.ts index ff94ac6ac..90b777ede 100644 --- a/tests/daily-cache-version-rederivation.test.ts +++ b/tests/daily-cache-version-rederivation.test.ts @@ -108,6 +108,99 @@ describe('daily-cache re-derivation on a DAILY_CACHE_VERSION bump', () => { expect(refreshedDay?.cost).toBe(2) expect(JSON.parse(await readFile(oldPath, 'utf8'))).toEqual(oldCache) }) + + it('refreshes finalized Codex categories on v51 migration without changing usage totals', async () => { + const date = toDateString(new Date(Date.now() - 14 * 24 * 60 * 60 * 1000)) + const yesterday = toDateString(new Date(Date.now() - 24 * 60 * 60 * 1000)) + const oldVersion = DAILY_CACHE_VERSION - 1 + const oldPath = join(cacheRoot, `daily-cache.v${oldVersion}.json`) + const makeCodexDay = (category: 'general' | 'exploration'): DailyEntry => { + const categoryStats = { turns: 1, cost: 2, savingsUSD: 0, editTurns: 0, oneShotTurns: 0 } + const modelStats = { + calls: 1, + cost: 2, + savingsUSD: 0, + inputTokens: 100, + outputTokens: 20, + cacheReadTokens: 30, + cacheWriteTokens: 0, + } + const providerSlice = { + calls: 1, + cost: 2, + savingsUSD: 0, + sessions: 1, + inputTokens: 100, + outputTokens: 20, + cacheReadTokens: 30, + cacheWriteTokens: 0, + models: { 'GPT-5.4': { ...modelStats } }, + categories: { [category]: { ...categoryStats } }, + } + return { + ...day(date, 2), + models: { 'GPT-5.4': { ...modelStats } }, + categories: { [category]: { ...categoryStats } }, + providers: { codex: providerSlice }, + } + } + const oldDay = makeCodexDay('general') + const freshDay = makeCodexDay('exploration') + const oldCache = { + version: oldVersion, + savingsConfigHash: '', + tzKey: currentTzKey(), + lastComputedDate: yesterday, + days: [oldDay], + complete: true, + watermarkTrusted: true, + } + await writeFile(oldPath, JSON.stringify(oldCache)) + + let parseCount = 0 + const hydrated = await ensureCacheHydrated( + async () => { + parseCount++ + return [] + }, + () => [freshDay], + '', + () => true, + ) + + const refreshedDay = hydrated.days.find(entry => entry.date === date) + expect(parseCount).toBe(1) + expect(hydrated.version).toBe(DAILY_CACHE_VERSION) + expect(refreshedDay?.categories).toEqual({ exploration: freshDay.categories.exploration }) + expect(refreshedDay?.providers.codex?.categories).toEqual({ exploration: freshDay.providers.codex.categories?.exploration }) + expect({ + calls: refreshedDay?.calls, + cost: refreshedDay?.cost, + savingsUSD: refreshedDay?.savingsUSD, + sessions: refreshedDay?.sessions, + inputTokens: refreshedDay?.inputTokens, + outputTokens: refreshedDay?.outputTokens, + cacheReadTokens: refreshedDay?.cacheReadTokens, + cacheWriteTokens: refreshedDay?.cacheWriteTokens, + provider: refreshedDay?.providers.codex, + models: refreshedDay?.models, + }).toEqual({ + calls: oldDay.calls, + cost: oldDay.cost, + savingsUSD: oldDay.savingsUSD, + sessions: oldDay.sessions, + inputTokens: oldDay.inputTokens, + outputTokens: oldDay.outputTokens, + cacheReadTokens: oldDay.cacheReadTokens, + cacheWriteTokens: oldDay.cacheWriteTokens, + provider: { + ...oldDay.providers.codex, + categories: freshDay.providers.codex.categories, + }, + models: oldDay.models, + }) + expect(JSON.parse(await readFile(oldPath, 'utf8'))).toEqual(oldCache) + }) }) // v30 is claimed by two open branches at once: this one and #1132. A v30 file diff --git a/tests/providers/codex.test.ts b/tests/providers/codex.test.ts index 32113cd9b..fcc70a814 100644 --- a/tests/providers/codex.test.ts +++ b/tests/providers/codex.test.ts @@ -1621,21 +1621,91 @@ describe('codex provider - native tool events', () => { expect(forkCalls[0]?.tools).toEqual(['WebSearch']) }) - it('attributes a large image-generation result through the compact line decoder', async () => { + it('keeps direct native identities in large lines without reading nested or sibling ids', async () => { const largeImage = JSON.stringify({ type: 'response_item', timestamp: '2026-04-14T10:00:10Z', - payload: { type: 'image_generation_call', id: 'ig-large', status: 'completed', revised_prompt: 'fixture', result: 'a'.repeat(2 * 1024 * 1024) }, + payload: { type: 'image_generation_call', status: 'completed', revised_prompt: 'fixture', result: 'a'.repeat(2 * 1024 * 1024), id: 'ig-late' }, + }) + const duplicateImage = JSON.stringify({ + type: 'response_item', timestamp: '2026-04-14T10:00:11Z', + payload: { type: 'image_generation_call', id: 'ig-late', status: 'completed', result: 'small' }, + }) + const imageWithoutId = JSON.stringify({ + type: 'response_item', timestamp: '2026-04-14T10:00:12Z', + payload: { type: 'image_generation_call', status: 'completed', result: 'b'.repeat(128 * 1024) }, + metadata: { id: 'ig-sibling' }, + }) + const imageSiblingControl = JSON.stringify({ + type: 'response_item', timestamp: '2026-04-14T10:00:13Z', + payload: { type: 'image_generation_call', id: 'ig-sibling', status: 'completed', result: 'small' }, + }) + const largeToolSearch = JSON.stringify({ + type: 'response_item', timestamp: '2026-04-14T10:00:14Z', + payload: { type: 'tool_search_call', arguments: { filler: 'c'.repeat(128 * 1024) }, call_id: 'ts-late' }, + }) + const duplicateToolSearch = JSON.stringify({ + type: 'response_item', timestamp: '2026-04-14T10:00:15Z', + payload: { type: 'tool_search_call', call_id: 'ts-late', arguments: {} }, + }) + const toolSearchWithoutId = JSON.stringify({ + type: 'response_item', timestamp: '2026-04-14T10:00:16Z', + payload: { type: 'tool_search_call', arguments: { filler: 'd'.repeat(128 * 1024) } }, + metadata: { call_id: 'ts-sibling' }, + }) + const toolSearchSiblingControl = JSON.stringify({ + type: 'response_item', timestamp: '2026-04-14T10:00:17Z', + payload: { type: 'tool_search_call', call_id: 'ts-sibling', arguments: {} }, + }) + const largeItemCompleted = JSON.stringify({ + type: 'event_msg', timestamp: '2026-04-14T10:00:18Z', + payload: { + type: 'item_completed', + item: { + results: [{ type: 'NestedResult', id: 'nested-result', body: 'e'.repeat(128 * 1024) }], + action: { type: 'search', id: 'nested-action' }, + type: 'WebSearch', + id: 'ws-late', + }, + }, + }) + const duplicateItemCompleted = JSON.stringify({ + type: 'event_msg', timestamp: '2026-04-14T10:00:19Z', + payload: { type: 'item_completed', item: { results: [], type: 'WebSearch', id: 'ws-late' } }, + }) + const itemWithoutId = JSON.stringify({ + type: 'event_msg', timestamp: '2026-04-14T10:00:20Z', + payload: { type: 'item_completed', item: { results: [{ type: 'NestedResult', id: 'nested-result' }], type: 'WebSearch' } }, + metadata: { item: { type: 'WebSearch', id: 'ws-sibling' } }, + }) + const itemSiblingControl = JSON.stringify({ + type: 'event_msg', timestamp: '2026-04-14T10:00:21Z', + payload: { type: 'item_completed', item: { type: 'WebSearch', id: 'ws-sibling' } }, }) const filePath = await writeSession(tmpDir, '2026-04-14', 'rollout-native-large-image.jsonl', [ sessionMeta({ session_id: 'sess-native-large-image' }), largeImage, + duplicateImage, + imageWithoutId, + imageSiblingControl, + largeToolSearch, + duplicateToolSearch, + toolSearchWithoutId, + toolSearchSiblingControl, + largeItemCompleted, + duplicateItemCompleted, + itemWithoutId, + itemSiblingControl, tokenUsageRecord({ responseId: 'resp-native-large-image', usage: { input: 100, output: 20 } }), ]) const calls = await parseFile(filePath) expect(calls).toHaveLength(1) - expect(calls[0]?.tools).toEqual(['ImageGeneration']) + expect(calls[0]?.tools).toEqual([ + 'ImageGeneration', 'ImageGeneration', 'ImageGeneration', + 'ToolSearch', 'ToolSearch', 'ToolSearch', + 'WebSearch', 'WebSearch', 'WebSearch', + ]) }) }) From 95526842fdc0d01379c83a2c70cf53dde5047944 Mon Sep 17 00:00:00 2001 From: Uttam Deb <105539415+uttamdeb@users.noreply.github.com> Date: Mon, 5 Oct 2026 11:20:26 +0600 Subject: [PATCH 4/8] Pin the shipped native-tool cache migration regression --- tests/daily-cache-version-rederivation.test.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/daily-cache-version-rederivation.test.ts b/tests/daily-cache-version-rederivation.test.ts index 90b777ede..0dd0c9c94 100644 --- a/tests/daily-cache-version-rederivation.test.ts +++ b/tests/daily-cache-version-rederivation.test.ts @@ -112,7 +112,9 @@ describe('daily-cache re-derivation on a DAILY_CACHE_VERSION bump', () => { it('refreshes finalized Codex categories on v51 migration without changing usage totals', async () => { const date = toDateString(new Date(Date.now() - 14 * 24 * 60 * 60 * 1000)) const yesterday = toDateString(new Date(Date.now() - 24 * 60 * 60 * 1000)) - const oldVersion = DAILY_CACHE_VERSION - 1 + // Pin the shipped version that missed native tools. Deriving this from + // DAILY_CACHE_VERSION would still pass if the required bump were removed. + const oldVersion = 51 const oldPath = join(cacheRoot, `daily-cache.v${oldVersion}.json`) const makeCodexDay = (category: 'general' | 'exploration'): DailyEntry => { const categoryStats = { turns: 1, cost: 2, savingsUSD: 0, editTurns: 0, oneShotTurns: 0 } From dea1b35457bb6cbcdf3f69f38e98c41612a21d05 Mon Sep 17 00:00:00 2001 From: Uttam Deb <105539415+uttamdeb@users.noreply.github.com> Date: Sat, 17 Oct 2026 23:57:18 +0600 Subject: [PATCH 5/8] Pin shipped v75 native-tool cache migration --- tests/daily-cache-version-rederivation.test.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/daily-cache-version-rederivation.test.ts b/tests/daily-cache-version-rederivation.test.ts index 0dd0c9c94..89cfad57c 100644 --- a/tests/daily-cache-version-rederivation.test.ts +++ b/tests/daily-cache-version-rederivation.test.ts @@ -109,12 +109,12 @@ describe('daily-cache re-derivation on a DAILY_CACHE_VERSION bump', () => { expect(JSON.parse(await readFile(oldPath, 'utf8'))).toEqual(oldCache) }) - it('refreshes finalized Codex categories on v51 migration without changing usage totals', async () => { + it('refreshes finalized Codex categories from a literal v75 cache without changing usage totals', async () => { const date = toDateString(new Date(Date.now() - 14 * 24 * 60 * 60 * 1000)) const yesterday = toDateString(new Date(Date.now() - 24 * 60 * 60 * 1000)) - // Pin the shipped version that missed native tools. Deriving this from - // DAILY_CACHE_VERSION would still pass if the required bump were removed. - const oldVersion = 51 + // Pin the shipped v75 cache that missed native tools. Deriving this from + // DAILY_CACHE_VERSION would still pass if the required v76 bump were removed. + const oldVersion = 75 const oldPath = join(cacheRoot, `daily-cache.v${oldVersion}.json`) const makeCodexDay = (category: 'general' | 'exploration'): DailyEntry => { const categoryStats = { turns: 1, cost: 2, savingsUSD: 0, editTurns: 0, oneShotTurns: 0 } From b71d8102ea198be6a80b11c2f3d69f9779243076 Mon Sep 17 00:00:00 2001 From: Uttam Deb <105539415+uttamdeb@users.noreply.github.com> Date: Sat, 17 Oct 2026 23:57:53 +0600 Subject: [PATCH 6/8] Test native event identity in fork replay bursts --- tests/providers/codex.test.ts | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/tests/providers/codex.test.ts b/tests/providers/codex.test.ts index fcc70a814..c5a409376 100644 --- a/tests/providers/codex.test.ts +++ b/tests/providers/codex.test.ts @@ -1621,6 +1621,29 @@ describe('codex provider - native tool events', () => { expect(forkCalls[0]?.tools).toEqual(['WebSearch']) }) + it('keeps a fork-only native identity before replay usage anchors and suppresses parent item ids', async () => { + const parentPath = await writeSession(tmpDir, '2026-10-17', 'rollout-2026-10-17T10-00-00-sess-native-parent.jsonl', [ + sessionMeta({ session_id: 'sess-native-parent', timestamp: '2026-10-17T10:00:00Z' }), + nativeResponseItem('web_search_call', { id: 'ws-parent', status: 'completed' }, '2026-10-17T10:00:01Z'), + tokenUsageRecord({ timestamp: '2026-10-17T10:00:02Z', responseId: 'resp-parent', usage: { input: 100, output: 20 } }), + ]) + const forkPath = await writeSession(tmpDir, '2026-10-17', 'rollout-2026-10-17T10-00-10-sess-native-fork.jsonl', [ + sessionMeta({ session_id: 'sess-native-fork', forked_from_id: 'sess-native-parent', timestamp: '2026-10-17T10:00:10Z' }), + nativeResponseItem('web_search_call', { id: 'ws-fork', status: 'completed' }, '2026-10-17T10:00:10.100Z'), + nativeResponseItem('web_search_call', { id: 'ws-parent', status: 'completed' }, '2026-10-17T10:00:10.200Z'), + tokenUsageRecord({ timestamp: '2026-10-17T10:00:10.300Z', responseId: 'resp-parent', usage: { input: 100, output: 20 } }), + tokenUsageRecord({ timestamp: '2026-10-17T10:00:10.400Z', responseId: 'resp-fork', usage: { input: 80, output: 15 } }), + ]) + + const parentCalls = await parseFile(parentPath) + const forkCalls = await parseFile(forkPath) + + expect(parentCalls[0]?.tools).toEqual(['WebSearch']) + expect(forkCalls).toHaveLength(1) + expect(forkCalls[0]?.tools).toEqual(['WebSearch']) + expect(forkCalls[0]?.inputTokens).toBe(80) + }) + it('keeps direct native identities in large lines without reading nested or sibling ids', async () => { const largeImage = JSON.stringify({ type: 'response_item', timestamp: '2026-04-14T10:00:10Z', From e7b5f7596efb78528e829b92bfc148f40a0f090a Mon Sep 17 00:00:00 2001 From: Uttam Deb <105539415+uttamdeb@users.noreply.github.com> Date: Sat, 17 Oct 2026 23:59:47 +0600 Subject: [PATCH 7/8] Test compact Codex payload identity isolation --- tests/providers/codex.test.ts | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/tests/providers/codex.test.ts b/tests/providers/codex.test.ts index c5a409376..f1f6f23c7 100644 --- a/tests/providers/codex.test.ts +++ b/tests/providers/codex.test.ts @@ -1730,6 +1730,29 @@ describe('codex provider - native tool events', () => { 'WebSearch', 'WebSearch', 'WebSearch', ]) }) + + it('does not classify a large compact line from a nested metadata payload', async () => { + const misleadingPayload = JSON.stringify({ + type: 'response_item', + timestamp: '2026-04-14T10:00:10Z', + metadata: { payload: { type: 'image_generation_call', id: 'ig-decoy' } }, + payload: { + type: 'message', + role: 'assistant', + content: [{ type: 'output_text', text: 'large message body '.repeat(5_000) }], + }, + }) + const filePath = await writeSession(tmpDir, '2026-04-14', 'rollout-native-nested-payload.jsonl', [ + sessionMeta({ session_id: 'sess-native-nested-payload' }), + misleadingPayload, + tokenUsageRecord({ responseId: 'resp-native-nested-payload', usage: { input: 100, output: 20 } }), + ]) + + const calls = await parseFile(filePath) + + expect(calls).toHaveLength(1) + expect(calls[0]?.tools).toEqual([]) + }) }) describe('codex provider - token_usage_record accounting', () => { From b2c3c360753dd229f6eefc24be30bc535aa1361c Mon Sep 17 00:00:00 2001 From: Uttam Deb <105539415+uttamdeb@users.noreply.github.com> Date: Sun, 18 Oct 2026 00:13:58 +0600 Subject: [PATCH 8/8] Preserve Codex native tool identity across forks --- docs/providers/codex.md | 13 ++-- src/codex-fork-replay.ts | 127 +++++++++++++++++++++++++++------- src/codex-jsonl-fields.ts | 83 ++++++++++++++++++++++ src/parse-worker.ts | 6 +- src/parse-workers.ts | 10 +-- src/providers/codex.ts | 126 ++++++++++++--------------------- tests/parse-workers.test.ts | 25 +++++-- tests/providers/codex.test.ts | 39 ++++++++++- 8 files changed, 302 insertions(+), 127 deletions(-) create mode 100644 src/codex-jsonl-fields.ts diff --git a/docs/providers/codex.md b/docs/providers/codex.md index 2d69cc2c2..4d336ca9c 100644 --- a/docs/providers/codex.md +++ b/docs/providers/codex.md @@ -76,18 +76,23 @@ Estimated events that fall back to char-counting use `codex:: `response:${responseId}` +export const codexReplayNativeToolIdentity = (tool: string, itemId: string): string => `native:${tool}:${itemId}` // Rollout names are `rollout-YYYY-MM-DDTHH-MM-SS-.jsonl`. const ROLLOUT_ID_OFFSET = 'rollout-YYYY-MM-DDTHH-MM-SS-'.length @@ -107,26 +109,94 @@ function codexHomeOf(path: string): string | undefined { return undefined } -type ParentUsage = { sig: string; records: Promise> } -const parentUsage = new Map() +type ParentReplayRecords = { sig: string; records: Promise> } +const parentReplayRecords = new Map() -async function readParentUsage(path: string): Promise> { +function nativeToolForResponseType(type: string | undefined): 'WebSearch' | 'ToolSearch' | 'ImageGeneration' | undefined { + if (type === 'web_search_call') return 'WebSearch' + if (type === 'tool_search_call') return 'ToolSearch' + if (type === 'image_generation_call') return 'ImageGeneration' + return undefined +} + +function addBufferParentReplayRecord(records: Array<[number, string]>, source: Buffer): void { + const entryType = getRawJsonStringFieldAtObject(source, 0, 'type') + if (!entryType) return + const timestamp = getRawJsonStringFieldAtObject(source, 0, 'timestamp') + const at = timestamp ? Date.parse(timestamp) : NaN + const payload = findRawObjectField(source, 0, 'payload') + if (!payload || source[payload.valueStart] !== 0x7b) return + const payloadStart = payload.valueStart + const payloadType = getRawJsonStringFieldAtObject(source, payloadStart, 'type') + + // Keep the original usage identity semantics exactly: historical counters + // still pass through JSON.parse, including when a large line is a Buffer. + if (entryType === 'token_usage_record' || (entryType === 'event_msg' && payloadType === 'token_count')) { + addParsedParentReplayRecord(records, source.toString('utf-8')) + return + } + + if (entryType === 'response_item') { + const tool = nativeToolForResponseType(payloadType) + if (!tool) return + const itemId = getRawJsonStringFieldAtObject(source, payloadStart, 'id') + ?? getRawJsonStringFieldAtObject(source, payloadStart, 'call_id') + if (itemId) records.push([at, codexReplayNativeToolIdentity(tool, itemId)]) + return + } + + if (entryType !== 'event_msg' || payloadType !== 'item_completed') return + const item = findRawObjectField(source, payloadStart, 'item') + if (!item || source[item.valueStart] !== 0x7b) return + const itemType = getRawJsonStringFieldAtObject(source, item.valueStart, 'type') + const itemKind = getRawJsonStringFieldAtObject(source, item.valueStart, 'kind') + if (itemType !== 'WebSearch' && !(itemType === 'Extension' && itemKind === 'web.search')) return + const itemId = getRawJsonStringFieldAtObject(source, item.valueStart, 'id') + if (itemId) records.push([at, codexReplayNativeToolIdentity('WebSearch', itemId)]) +} + +function addParsedParentReplayRecord(records: Array<[number, string]>, line: string): void { + let entry: { type?: string; timestamp?: string; payload?: { type?: string; response_id?: unknown; id?: unknown; call_id?: unknown; info?: Parameters[0]; item?: { type?: unknown; id?: unknown; kind?: unknown } } } + try { entry = JSON.parse(line) } catch { return } + const at = entry.timestamp ? Date.parse(entry.timestamp) : NaN + if (entry.type === 'token_usage_record' && typeof entry.payload?.response_id === 'string') { + records.push([at, codexReplayResponseIdentity(entry.payload.response_id)]) + } else if (entry.type === 'event_msg' && entry.payload?.type === 'token_count' && entry.payload.info) { + records.push([at, codexReplayUsageIdentity(entry.payload.info)]) + } else if (entry.type === 'response_item') { + const tool = nativeToolForResponseType(entry.payload?.type) + const itemId = typeof entry.payload?.id === 'string' && entry.payload.id + ? entry.payload.id + : typeof entry.payload?.call_id === 'string' && entry.payload.call_id + ? entry.payload.call_id + : undefined + if (tool && itemId) records.push([at, codexReplayNativeToolIdentity(tool, itemId)]) + } else if (entry.type === 'event_msg' && entry.payload?.type === 'item_completed' + && (entry.payload.item?.type === 'WebSearch' + || (entry.payload.item?.type === 'Extension' && entry.payload.item.kind === 'web.search')) + && typeof entry.payload.item.id === 'string' && entry.payload.item.id) { + records.push([at, codexReplayNativeToolIdentity('WebSearch', entry.payload.item.id)]) + } +} + +async function readParentReplayRecords(path: string): Promise> { const records: Array<[number, string]> = [] - const skip = (head: string) => !head.includes('"token_count"') && !head.includes('"token_usage_record"') - for await (const line of readSessionLines(path, skip)) { - let entry: { type?: string; timestamp?: string; payload?: { type?: string; response_id?: unknown; info?: Parameters[0] } } - try { entry = JSON.parse(line) } catch { continue } - const at = entry.timestamp ? Date.parse(entry.timestamp) : NaN - if (entry.type === 'token_usage_record' && typeof entry.payload?.response_id === 'string') { - records.push([at, codexReplayResponseIdentity(entry.payload.response_id)]) - } else if (entry.type === 'event_msg' && entry.payload?.type === 'token_count' && entry.payload.info) { - records.push([at, codexReplayUsageIdentity(entry.payload.info)]) - } + const skip = (head: string) => !head.includes('"token_count"') + && !head.includes('"token_usage_record"') + && !head.includes('"web_search_call"') + && !head.includes('"tool_search_call"') + && !head.includes('"image_generation_call"') + && !head.includes('"item_completed"') + for await (const line of readSessionLines(path, skip, { largeLineAsBuffer: true })) { + if (Buffer.isBuffer(line)) addBufferParentReplayRecord(records, line) + else addParsedParentReplayRecord(records, line) } return records } -type ParentReplayLoader = (forkPath: string, parentId: string, forkedAtMs: number) => Promise | null> +export type CodexParentReplayIndex = { usageIdentities: Set; nativeIdentities: Set } + +type ParentReplayLoader = (forkPath: string, parentId: string, forkedAtMs: number) => Promise let parentReplayRequest: ParentReplayLoader | null = null /** A parse worker hands the lookup to the parent thread, so each parent is indexed and read once, not once per worker. */ @@ -135,12 +205,12 @@ export function routeCodexParentReplay(request: ParentReplayLoader): void { } /** - * Usage identities the parent rollout recorded up to the fork, read from the - * parent's raw file (its own copied history included, so a chain resolves one - * level at a time). Null when the parent is not on disk next to the fork; the - * caller then drops the whole replay burst. + * Usage and native item identities the parent rollout recorded up to the fork, + * read from the parent's raw file (its own copied history included, so a chain + * resolves one level at a time). A null result means the parent is not readable + * next to the fork; an empty native set means it was read and contains no items. */ -export async function loadCodexParentReplay(forkPath: string, parentId: string, forkedAtMs: number): Promise | null> { +export async function loadCodexParentReplayIndex(forkPath: string, parentId: string, forkedAtMs: number): Promise { if (parentReplayRequest) return parentReplayRequest(forkPath, parentId, forkedAtMs) const home = codexHomeOf(forkPath) const path = home ? findRollout(home, parentId) : undefined @@ -152,12 +222,21 @@ export async function loadCodexParentReplay(forkPath: string, parentId: string, } catch { return null } - let cached = parentUsage.get(path) + let cached = parentReplayRecords.get(path) if (cached?.sig !== sig) { - cached = { sig, records: readParentUsage(path) } - parentUsage.set(path, cached) + cached = { sig, records: readParentReplayRecords(path) } + parentReplayRecords.set(path, cached) } // Records the parent wrote after the fork were never copied into it. - const ids = new Set((await cached.records).filter(([at]) => !(at > forkedAtMs)).map(([, id]) => id)) - return ids.size > 0 ? ids : null + const records = (await cached.records).filter(([at]) => !(at > forkedAtMs)) + return { + usageIdentities: new Set(records.map(([, id]) => id).filter(id => !id.startsWith('native:'))), + nativeIdentities: new Set(records.map(([, id]) => id).filter(id => id.startsWith('native:'))), + } +} + +/** Usage-only compatibility wrapper; an empty identity set still means no replay anchor. */ +export async function loadCodexParentReplay(forkPath: string, parentId: string, forkedAtMs: number): Promise | null> { + const index = await loadCodexParentReplayIndex(forkPath, parentId, forkedAtMs) + return index?.usageIdentities.size ? index.usageIdentities : null } diff --git a/src/codex-jsonl-fields.ts b/src/codex-jsonl-fields.ts new file mode 100644 index 000000000..8d91737b2 --- /dev/null +++ b/src/codex-jsonl-fields.ts @@ -0,0 +1,83 @@ +export type RawJsonFieldBounds = { keyStart: number; valueStart: number } + +/** Find a direct property in a JSON object without decoding its values. */ +export function findRawObjectField(source: Buffer, objectStart: number, field: string): RawJsonFieldBounds | undefined { + const fieldBytes = Buffer.from(field, 'utf-8') + let depth = 0 + let rootStarted = false + let inString = false + let escaped = false + for (let i = objectStart; i < source.length; i++) { + const byte = source[i]! + if (!rootStarted) { + if (byte === 0x20 || byte === 0x09 || byte === 0x0a || byte === 0x0d) continue + if (byte !== 0x7b) return undefined // { + rootStarted = true + depth = 1 + continue + } + if (inString) { + if (escaped) escaped = false + else if (byte === 0x5c) escaped = true // \\ + else if (byte === 0x22) inString = false // " + continue + } + if (byte === 0x22) { + const keyStart = i + 1 + let keyEnd = keyStart + let keyEscaped = false + for (; keyEnd < source.length; keyEnd++) { + const keyByte = source[keyEnd]! + if (keyEscaped) { keyEscaped = false; continue } + if (keyByte === 0x5c) { keyEscaped = true; continue } + if (keyByte === 0x22) break + } + if (depth === 1 && keyEnd < source.length) { + // Most strings at depth 1 are values, including potentially huge + // image results. Check the key/value delimiter and exact short-key + // length before comparing bytes so scanning never materializes them. + let colon = keyEnd + 1 + while (colon < source.length && (source[colon] === 0x20 || source[colon] === 0x09 || source[colon] === 0x0a || source[colon] === 0x0d)) colon++ + if (source[colon] === 0x3a && keyEnd - keyStart === fieldBytes.length && source.subarray(keyStart, keyEnd).equals(fieldBytes)) { + let valueStart = colon + 1 + while (valueStart < source.length && (source[valueStart] === 0x20 || source[valueStart] === 0x09 || source[valueStart] === 0x0a || source[valueStart] === 0x0d)) valueStart++ + return { keyStart: i, valueStart } + } + } + i = keyEnd + inString = false + continue + } + if (byte === 0x7b || byte === 0x5b) depth++ // { or [ + else if (byte === 0x7d || byte === 0x5d) depth-- // } or ] + if (depth <= 0) break + } + return undefined +} + +export function getRawObjectFieldWindow(source: Buffer, objectStart: number, field: string, windowBytes = 4096): string | undefined { + const bounds = findRawObjectField(source, objectStart, field) + return bounds ? source.subarray(bounds.keyStart, Math.min(source.length, bounds.keyStart + windowBytes)).toString('utf-8') : undefined +} + +/** Read a short direct string property while bounding how much is decoded. */ +export function getRawJsonStringFieldAtObject(source: Buffer, objectStart: number, field: string, maxBytes = 4096): string | undefined { + const bounds = findRawObjectField(source, objectStart, field) + if (!bounds || source[bounds.valueStart] !== 0x22) return undefined + + let escaped = false + const valueEndLimit = Math.min(source.length, bounds.valueStart + maxBytes + 1) + for (let i = bounds.valueStart + 1; i < valueEndLimit; i++) { + const byte = source[i]! + if (escaped) escaped = false + else if (byte === 0x5c) escaped = true + else if (byte === 0x22) { + try { + return JSON.parse(source.subarray(bounds.valueStart, i + 1).toString('utf-8')) as string + } catch { + return undefined + } + } + } + return undefined +} diff --git a/src/parse-worker.ts b/src/parse-worker.ts index 105b350f3..7d4018823 100644 --- a/src/parse-worker.ts +++ b/src/parse-worker.ts @@ -1,5 +1,5 @@ import { parentPort, workerData, type MessagePort } from 'worker_threads' -import { routeCodexParentReplay } from './codex-fork-replay.js' +import { routeCodexParentReplay, type CodexParentReplayIndex } from './codex-fork-replay.js' import { restorePricingState, type PricingSnapshot } from './models.js' import type { ParentReplayReply, ParentReplayRequest, ParseJob } from './parse-workers.js' import { parseClaudeFileFull } from './parser.js' @@ -11,13 +11,13 @@ if (!port) throw new Error('parse-worker must be started as a worker thread') const { pricing, replayPort } = workerData as { pricing: PricingSnapshot; replayPort: MessagePort } restorePricingState(pricing) -const replayReplies = new Map | null) => void; reject: (err: Error) => void }>() +const replayReplies = new Map void; reject: (err: Error) => void }>() let nextReplayRequest = 0 replayPort.on('message', (msg: ParentReplayReply) => { const reply = replayReplies.get(msg.id) replayReplies.delete(msg.id) if (msg.error !== undefined) reply?.reject(new Error(msg.error)) - else reply?.resolve(msg.ids ?? null) + else reply?.resolve(msg.index ?? null) }) routeCodexParentReplay((...args) => new Promise((resolve, reject) => { const id = nextReplayRequest++ diff --git a/src/parse-workers.ts b/src/parse-workers.ts index e771976cd..3a80e4da3 100644 --- a/src/parse-workers.ts +++ b/src/parse-workers.ts @@ -1,6 +1,6 @@ import { availableParallelism, totalmem } from 'os' import { MessageChannel, Worker, type MessagePort } from 'worker_threads' -import { loadCodexParentReplay } from './codex-fork-replay.js' +import { loadCodexParentReplayIndex, type CodexParentReplayIndex } from './codex-fork-replay.js' import { snapshotPricingState } from './models.js' import type { ClaudeFileParse } from './parser.js' import type { SessionSource } from './providers/types.js' @@ -124,13 +124,13 @@ type Task = { job: ParseJob; resolve: (r: ParseWorkerResult) => void } type WorkerMessage = { json?: string | null; error?: string } -export type ParentReplayRequest = { id: number; args: Parameters } -export type ParentReplayReply = { id: number; ids?: Set | null; error?: string } +export type ParentReplayRequest = { id: number; args: Parameters } +export type ParentReplayReply = { id: number; index?: CodexParentReplayIndex | null; error?: string } function serveParentReplay(port: MessagePort): void { port.on('message', ({ id, args }: ParentReplayRequest) => { - loadCodexParentReplay(...args).then( - ids => port.postMessage({ id, ids } satisfies ParentReplayReply), + loadCodexParentReplayIndex(...args).then( + index => port.postMessage({ id, index } satisfies ParentReplayReply), (err: unknown) => port.postMessage({ id, error: err instanceof Error ? err.message : String(err) } satisfies ParentReplayReply), ) }) diff --git a/src/providers/codex.ts b/src/providers/codex.ts index ceb5aed60..ecfcdf3e8 100644 --- a/src/providers/codex.ts +++ b/src/providers/codex.ts @@ -8,7 +8,8 @@ import { FS_SCAN_CONCURRENCY, mapWithConcurrency, readSessionLines } from '../fs import { billableOutputTokens, calculateCost, getModelCosts, pricingModelAt } from '../models.js' import { readCachedCodexResults, writeCachedCodexResults, getCachedCodexProject, fingerprintFile, type CodexFileFingerprint } from '../codex-cache.js' import { mergeToolIntervals } from '../codex-throughput.js' -import { codexReplayResponseIdentity, codexReplayUsageIdentity, isCodexForkReplay, isCodexForkReplayState, loadCodexParentReplay, startCodexForkReplay, type CodexForkReplayState } from '../codex-fork-replay.js' +import { codexReplayNativeToolIdentity, codexReplayResponseIdentity, codexReplayUsageIdentity, isCodexForkReplay, isCodexForkReplayState, loadCodexParentReplayIndex, startCodexForkReplay, type CodexForkReplayState } from '../codex-fork-replay.js' +import { findRawObjectField, getRawObjectFieldWindow } from '../codex-jsonl-fields.js' import { normalizeContentBlocks } from '../content-utils.js' import { estimateTokensFromChars } from '../token-estimate.js' import { wslHomes } from '../wsl.js' @@ -200,13 +201,14 @@ function nativeToolEvent(entry: CodexEntry): CodexNativeToolEvent | null { : undefined } else if (entry.type === 'event_msg' && payload.type === 'item_completed') { const item = payload.item - if (!item || item['type'] !== 'WebSearch') return null + if (!item) return null + if (item['type'] !== 'WebSearch' && !(item['type'] === 'Extension' && item['kind'] === 'web.search')) return null tool = 'WebSearch' identity = typeof item['id'] === 'string' && item['id'] ? item['id'] : undefined } if (!tool) return null - return { tool, ...(identity ? { identity: `${tool}:${identity}` } : {}) } + return { tool, ...(identity ? { identity: codexReplayNativeToolIdentity(tool, identity) } : {}) } } type CodexTokenUsage = { @@ -320,67 +322,9 @@ function getRawJsonNumberField(head: string, field: string): number | undefined return Number.isFinite(value) ? value : undefined } -type RawJsonFieldBounds = { keyStart: number; valueStart: number } - -function findRawObjectField(source: Buffer, objectStart: number, field: string): RawJsonFieldBounds | undefined { - const fieldBytes = Buffer.from(field, 'utf-8') - let depth = 0 - let inString = false - let escaped = false - for (let i = objectStart; i < source.length; i++) { - const byte = source[i]! - if (inString) { - if (escaped) escaped = false - else if (byte === 0x5c) escaped = true // \\ - else if (byte === 0x22) inString = false // " - continue - } - if (byte === 0x22) { - const keyStart = i + 1 - let keyEnd = keyStart - let keyEscaped = false - for (; keyEnd < source.length; keyEnd++) { - const keyByte = source[keyEnd]! - if (keyEscaped) { keyEscaped = false; continue } - if (keyByte === 0x5c) { keyEscaped = true; continue } - if (keyByte === 0x22) break - } - if (depth === 1 && keyEnd < source.length) { - // Most strings at depth 1 are values, including potentially huge - // image results. Check for the key/value delimiter and exact short-key - // length before comparing bytes so scanning never materializes them. - let colon = keyEnd + 1 - while (colon < source.length && (source[colon] === 0x20 || source[colon] === 0x09 || source[colon] === 0x0a || source[colon] === 0x0d)) colon++ - if (source[colon] === 0x3a && keyEnd - keyStart === fieldBytes.length && source.subarray(keyStart, keyEnd).equals(fieldBytes)) { - let valueStart = colon - valueStart++ - while (valueStart < source.length && (source[valueStart] === 0x20 || source[valueStart] === 0x09 || source[valueStart] === 0x0a || source[valueStart] === 0x0d)) valueStart++ - return { keyStart: i, valueStart } - } - } - i = keyEnd - inString = false - continue - } - if (byte === 0x22) inString = true - else if (byte === 0x7b || byte === 0x5b) depth++ // { or [ - else if (byte === 0x7d || byte === 0x5d) depth-- // } or ] - if (depth <= 0) break - } - return undefined -} - -function getRawObjectFieldWindow(source: Buffer, objectStart: number, field: string, windowBytes = 4096): string | undefined { - const bounds = findRawObjectField(source, objectStart, field) - return bounds ? source.subarray(bounds.keyStart, Math.min(source.length, bounds.keyStart + windowBytes)).toString('utf-8') : undefined -} - function getRawPayloadObjectStart(source: Buffer): number | undefined { - const payloadKey = Buffer.from('"payload"') - const payloadIndex = source.indexOf(payloadKey) - if (payloadIndex < 0) return undefined - const payloadStart = source.indexOf(0x7b, payloadIndex + payloadKey.length) // { - return payloadStart < 0 ? undefined : payloadStart + const payload = findRawObjectField(source, 0, 'payload') + return payload && source[payload.valueStart] === 0x7b ? payload.valueStart : undefined } function getRawPayloadFieldWindow(source: Buffer, field: string, windowBytes = 4096): string | undefined { @@ -448,9 +392,11 @@ function getRawTokenUsage(head: string, field: 'last_token_usage' | 'total_token } } -function payloadHead(head: string): string { - const idx = head.indexOf('"payload"') - return idx === -1 ? head : head.slice(idx) +function payloadHead(source: Buffer): string { + const payloadStart = getRawPayloadObjectStart(source) + return payloadStart === undefined + ? '' + : source.subarray(payloadStart, Math.min(source.length, payloadStart + RAW_HEAD_BYTES)).toString('utf-8') } function getRawInvocation(head: string): { server?: string; tool?: string } | undefined { @@ -555,8 +501,9 @@ function parseCodexLine(line: string | Buffer): CodexEntry | null { const head = line.subarray(0, RAW_HEAD_BYTES).toString('utf-8') const type = getRawJsonStringField(head, 'type') if (!type) return null - const pHead = payloadHead(head) - const payloadType = getRawJsonStringField(pHead, 'type') + const pHead = payloadHead(line) + const payloadTypeWindow = getRawPayloadFieldWindow(line, 'type') + const payloadType = payloadTypeWindow ? getRawJsonStringField(payloadTypeWindow, 'type') : undefined const role = getRawJsonStringField(pHead, 'role') // task_complete appends the potentially huge final assistant message before // its duration fields. Fall back to the full Buffer only for this event so @@ -594,10 +541,14 @@ function parseCodexLine(line: string | Buffer): CodexEntry | null { const itemIdWindow = type === 'event_msg' && payloadType === 'item_completed' ? getRawPayloadItemFieldWindow(line, 'id') : undefined + const itemKindWindow = type === 'event_msg' && payloadType === 'item_completed' + ? getRawPayloadItemFieldWindow(line, 'kind') + : undefined const compactItemType = itemTypeWindow ? getRawJsonStringField(itemTypeWindow, 'type') : undefined const compactItemId = itemIdWindow ? getRawJsonStringField(itemIdWindow, 'id') : undefined + const compactItemKind = itemKindWindow ? getRawJsonStringField(itemKindWindow, 'kind') : undefined const compactItem = compactItemType - ? { type: compactItemType, ...(compactItemId ? { id: compactItemId } : {}) } + ? { type: compactItemType, ...(compactItemId ? { id: compactItemId } : {}), ...(compactItemKind ? { kind: compactItemKind } : {}) } : undefined const isNativeResponseTool = type === 'response_item' && ( payloadType === 'web_search_call' || payloadType === 'tool_search_call' || payloadType === 'image_generation_call' @@ -925,16 +876,22 @@ function createParser(source: SessionSource, seenKeys: Set, capture?: { let sessionCwd: string | undefined = resume?.state.sessionCwd let forkedFromId = resume?.state.forkedFromId ?? '' let forkReplayState = resume?.state.forkReplayState ? { ...resume.state.forkReplayState } : undefined - // What the parent rollout recorded before the fork: a burst record found - // there is a copy and is dropped. The rest is the parent's work that its - // rollout kept only inside a running total, so it counts here. Loaded on - // the first burst usage record; null drops the whole burst. + // What the parent rollout recorded before the fork. Usage records use + // their conservative replay anchor; native items use exact ID membership. + // Loaded on the first relevant record; null drops the whole replay burst. let parentReplay: Set | null | undefined + let parentNativeReplay: Set | null | undefined let parentReplayAnchored = false + const loadParentReplay = async (): Promise => { + if (parentReplay !== undefined) return + const index = forkedFromId && forkReplayState + ? await loadCodexParentReplayIndex(source.path, forkedFromId, forkReplayState.startedAtMs) + : null + parentReplay = index?.usageIdentities.size ? index.usageIdentities : null + parentNativeReplay = index?.nativeIdentities ?? null + } const isParentReplay = async (identity: string | undefined): Promise => { - if (parentReplay === undefined) { - parentReplay = forkedFromId && forkReplayState ? await loadCodexParentReplay(source.path, forkedFromId, forkReplayState.startedAtMs) : null - } + await loadParentReplay() if (!parentReplay || identity === undefined) return true if (parentReplay.has(identity)) { parentReplayAnchored = true; return true } // A burst that opens with a record the parent lacks was not copied from @@ -942,6 +899,13 @@ function createParser(source: SessionSource, seenKeys: Set, capture?: { if (!parentReplayAnchored) { parentReplay = null; return true } return false } + const isParentNativeReplay = async (identity: string | undefined): Promise => { + await loadParentReplay() + // Without a readable parent set or a stable item ID, retain the old + // burst suppression. A loaded set makes membership an exact decision, + // independent of whether a token record has anchored the fork yet. + return !parentNativeReplay || identity === undefined || parentNativeReplay.has(identity) + } // Null sentinel rather than `0` so the FIRST event is never confused // with a duplicate. A session that only emits last_token_usage (no // total_token_usage) reports cumulativeTotal=0 on every event; with a @@ -1197,6 +1161,7 @@ function createParser(source: SessionSource, seenKeys: Set, capture?: { || entry.payload?.source?.subagent?.thread_spawn?.parent_thread_id || '' parentReplay = undefined + parentNativeReplay = undefined parentReplayAnchored = false if (forkedFromId) { forkReplayState = startCodexForkReplay(entry.timestamp) @@ -1226,6 +1191,9 @@ function createParser(source: SessionSource, seenKeys: Set, capture?: { } const isForkReplay = isCodexForkReplay(forkReplayState, entry.timestamp) + const nativeEvent = nativeToolEvent(entry) + + if (nativeEvent && isForkReplay && await isParentNativeReplay(nativeEvent.identity)) continue if (entry.type === 'turn_context' && typeof entry.payload?.model === 'string') { sessionModel = entry.payload.model @@ -1253,11 +1221,8 @@ function createParser(source: SessionSource, seenKeys: Set, capture?: { entry.payload?.type === 'function_call_output' || entry.payload?.type === 'custom_tool_call' || entry.payload?.type === 'custom_tool_call_output' || - entry.payload?.type === 'web_search_call' || - entry.payload?.type === 'tool_search_call' || - entry.payload?.type === 'image_generation_call' || entry.payload?.type === 'mcp_tool_call_end' || - entry.payload?.type === 'item_completed' || + (entry.payload?.type === 'item_completed' && !nativeEvent) || entry.payload?.type === 'patch_apply_end' )) continue @@ -1340,7 +1305,6 @@ function createParser(source: SessionSource, seenKeys: Set, capture?: { continue } - const nativeEvent = nativeToolEvent(entry) if (nativeEvent) { // Status transitions share an item identity. The first observed // record proves tool activity even when the rollout ends before its diff --git a/tests/parse-workers.test.ts b/tests/parse-workers.test.ts index 13c4bc44b..37520fc51 100644 --- a/tests/parse-workers.test.ts +++ b/tests/parse-workers.test.ts @@ -6,7 +6,7 @@ import { createHash } from 'node:crypto' import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' -import { loadCodexParentReplay } from '../src/codex-fork-replay.js' +import { loadCodexParentReplayIndex } from '../src/codex-fork-replay.js' import { decideParseWorkers, ParseWorkerPool, parseFilesInOrder, type ClaudeWorkerParse } from '../src/parse-workers.js' import { clearSessionCache, parseAllSessions, parseClaudeFileFull } from '../src/parser.js' import { parseCodexFileFull, type CodexFullParse } from '../src/providers/codex.js' @@ -15,7 +15,7 @@ import type { SessionSource } from '../src/providers/types.js' // Pass-through, so the test can see a worker's fork-parent lookup land on this thread. vi.mock('../src/codex-fork-replay.js', async (importOriginal) => { const actual = await importOriginal() - return { ...actual, loadCodexParentReplay: vi.fn(actual.loadCodexParentReplay) } + return { ...actual, loadCodexParentReplayIndex: vi.fn(actual.loadCodexParentReplayIndex) } }) // Two full cold CLI parses of a multi-hundred-file corpus, plus in-process parses @@ -491,8 +491,8 @@ describe('ParseWorkerPool', () => { }) // Every thread used to index and read a fork's parent rollout itself; the - // lookup now runs once on this thread, and the masking must not change. - it('resolves a fork parent on the parent thread and masks the burst as the serial path does', async () => { + // lookup now runs once on this thread, and usage/native masking must match serial parsing. + it('resolves fork usage and native identities on the parent thread', async () => { const parentId = '019e0000-0000-7000-8000-0000000000aa' const codexHome = join(home, '.codex') const meta = (id: string, timestamp: string, forked?: string) => JSON.stringify({ @@ -503,11 +503,21 @@ describe('ParseWorkerPool', () => { type: 'event_msg', timestamp, payload: { type: 'token_count', info: { last_token_usage: { input_tokens: input, total_tokens: input }, total_token_usage: { input_tokens: total, total_tokens: total } } }, }) - await writeCodexRollout(codexHome, '04', `2026-05-04T09-00-00-${parentId}`, [meta(parentId, '2026-05-04T09:00:00.000Z'), tokens('2026-05-04T09:00:01.000Z', 1000, 1000)].join('\n') + '\n') + const native = (timestamp: string, id: string) => JSON.stringify({ + type: 'response_item', timestamp, + payload: { type: 'web_search_call', id, status: 'completed' }, + }) + await writeCodexRollout(codexHome, '04', `2026-05-04T09-00-00-${parentId}`, [ + meta(parentId, '2026-05-04T09:00:00.000Z'), + native('2026-05-04T09:00:01.000Z', 'ws-parent'), + tokens('2026-05-04T09:00:02.000Z', 1000, 1000), + ].join('\n') + '\n') const sources: SessionSource[] = [] for (const n of [1, 2]) { const path = await writeCodexRollout(codexHome, '04', `fork-${n}`, [ meta(`fork-${n}`, '2026-05-04T10:00:00.000Z', parentId), + native('2026-05-04T10:00:00.001Z', `ws-fork-${n}`), + native('2026-05-04T10:00:00.002Z', 'ws-parent'), tokens('2026-05-04T10:00:00.005Z', 1000, 1000), // Inside the replay burst but absent from the parent: counted only when the parent was found. tokens('2026-05-04T10:00:00.010Z', 500, 1500), @@ -516,18 +526,19 @@ describe('ParseWorkerPool', () => { sources.push({ provider: 'codex', path, project: 'tmp-cxf' }) } - vi.mocked(loadCodexParentReplay).mockClear() + vi.mocked(loadCodexParentReplayIndex).mockClear() const pool = new ParseWorkerPool(2) const results = [] for await (const r of parseFilesInOrder(pool, sources.map(source => ({ kind: 'codex' as const, source })))) results.push(r) await pool.close() - expect(vi.mocked(loadCodexParentReplay)).toHaveBeenCalledTimes(2) + expect(vi.mocked(loadCodexParentReplayIndex)).toHaveBeenCalledTimes(2) for (const [i, source] of sources.entries()) { const r = results[i]! if (!r.ok || !r.parsed) throw new Error('expected a parsed result') const { keys: _keys, path: _path, ...worker } = r.parsed expect(worker.calls.map(c => c.inputTokens)).toEqual([500, 700]) + expect(worker.calls.map(c => c.tools)).toEqual([['WebSearch'], []]) expect(worker).toEqual(JSON.parse(JSON.stringify(await parseCodexFileFull(source, new Set())))) } }) diff --git a/tests/providers/codex.test.ts b/tests/providers/codex.test.ts index f1f6f23c7..1e9acd105 100644 --- a/tests/providers/codex.test.ts +++ b/tests/providers/codex.test.ts @@ -1536,6 +1536,16 @@ describe('codex provider - native tool events', () => { }) } + function completedExtension(kind: string, id: string, timestamp: string, results: unknown[] = []) { + return JSON.stringify({ + type: 'event_msg', timestamp, + payload: { + type: 'item_completed', + item: { type: 'Extension', results, id, action: { type: 'search' }, kind, query: 'fixture query' }, + }, + }) + } + async function parseFile(filePath: string): Promise { const provider = createCodexProvider(tmpDir) const calls: ParsedProviderCall[] = [] @@ -1585,6 +1595,20 @@ describe('codex provider - native tool events', () => { expect(call?.toolSequence?.map(step => step[0]?.tool)).toEqual(['WebSearch', 'WebSearch']) }) + it('recognizes only Extension items with the explicit web.search kind', async () => { + const largeResults = [{ body: 'fixture result body '.repeat(3_000) }] + const filePath = await writeSession(tmpDir, '2026-04-14', 'rollout-native-extension-search.jsonl', [ + sessionMeta({ session_id: 'sess-native-extension-search' }), + completedExtension('web.search', 'ext-search', '2026-04-14T10:00:10Z', largeResults), + completedExtension('file.search', 'ext-other', '2026-04-14T10:00:11Z'), + tokenUsageRecord({ responseId: 'resp-native-extension-search', usage: { input: 100, output: 20 } }), + ]) + + const [call] = await parseFile(filePath) + + expect(call?.tools).toEqual(['WebSearch']) + }) + it('attaches native tool events after the final usage record to that turn', async () => { const filePath = await writeSession(tmpDir, '2026-04-14', 'rollout-native-trailing.jsonl', [ sessionMeta({ session_id: 'sess-native-trailing' }), @@ -1622,15 +1646,24 @@ describe('codex provider - native tool events', () => { }) it('keeps a fork-only native identity before replay usage anchors and suppresses parent item ids', async () => { + const largeParentImage = ` \t${JSON.stringify({ + type: 'response_item', + timestamp: '2026-10-17T10:00:01.500Z', + payload: { type: 'image_generation_call', status: 'completed', result: 'x'.repeat(128 * 1024), id: 'ig-parent' }, + })}` const parentPath = await writeSession(tmpDir, '2026-10-17', 'rollout-2026-10-17T10-00-00-sess-native-parent.jsonl', [ sessionMeta({ session_id: 'sess-native-parent', timestamp: '2026-10-17T10:00:00Z' }), nativeResponseItem('web_search_call', { id: 'ws-parent', status: 'completed' }, '2026-10-17T10:00:01Z'), + completedExtension('web.search', 'ext-parent', '2026-10-17T10:00:01.250Z', [{ body: 'parent extension body '.repeat(3_000) }]), + largeParentImage, tokenUsageRecord({ timestamp: '2026-10-17T10:00:02Z', responseId: 'resp-parent', usage: { input: 100, output: 20 } }), ]) const forkPath = await writeSession(tmpDir, '2026-10-17', 'rollout-2026-10-17T10-00-10-sess-native-fork.jsonl', [ sessionMeta({ session_id: 'sess-native-fork', forked_from_id: 'sess-native-parent', timestamp: '2026-10-17T10:00:10Z' }), nativeResponseItem('web_search_call', { id: 'ws-fork', status: 'completed' }, '2026-10-17T10:00:10.100Z'), nativeResponseItem('web_search_call', { id: 'ws-parent', status: 'completed' }, '2026-10-17T10:00:10.200Z'), + nativeResponseItem('image_generation_call', { id: 'ig-parent', status: 'completed', result: 'small' }, '2026-10-17T10:00:10.250Z'), + completedExtension('web.search', 'ext-parent', '2026-10-17T10:00:10.275Z'), tokenUsageRecord({ timestamp: '2026-10-17T10:00:10.300Z', responseId: 'resp-parent', usage: { input: 100, output: 20 } }), tokenUsageRecord({ timestamp: '2026-10-17T10:00:10.400Z', responseId: 'resp-fork', usage: { input: 80, output: 15 } }), ]) @@ -1638,7 +1671,7 @@ describe('codex provider - native tool events', () => { const parentCalls = await parseFile(parentPath) const forkCalls = await parseFile(forkPath) - expect(parentCalls[0]?.tools).toEqual(['WebSearch']) + expect(parentCalls[0]?.tools).toEqual(['WebSearch', 'WebSearch', 'ImageGeneration']) expect(forkCalls).toHaveLength(1) expect(forkCalls[0]?.tools).toEqual(['WebSearch']) expect(forkCalls[0]?.inputTokens).toBe(80) @@ -1732,7 +1765,7 @@ describe('codex provider - native tool events', () => { }) it('does not classify a large compact line from a nested metadata payload', async () => { - const misleadingPayload = JSON.stringify({ + const misleadingPayload = ` \t${JSON.stringify({ type: 'response_item', timestamp: '2026-04-14T10:00:10Z', metadata: { payload: { type: 'image_generation_call', id: 'ig-decoy' } }, @@ -1741,7 +1774,7 @@ describe('codex provider - native tool events', () => { role: 'assistant', content: [{ type: 'output_text', text: 'large message body '.repeat(5_000) }], }, - }) + })}` const filePath = await writeSession(tmpDir, '2026-04-14', 'rollout-native-nested-payload.jsonl', [ sessionMeta({ session_id: 'sess-native-nested-payload' }), misleadingPayload,