Attribute native Codex tool events

docs/providers/codex.md

@@ -71,6 +71,29 @@ Four layers, in order:
 
 Estimated events that fall back to char-counting use `codex:<sessionId>:<timestamp>:est<n>`.
 
+## Native tool attribution
+
+`web_search_call`, `tool_search_call`, and `image_generation_call` are recorded
+as `WebSearch`, `ToolSearch`, and `ImageGeneration`. `event_msg/item_completed`
+records whose item type is `WebSearch` share the same `WebSearch` identity.
+Persisted WebSearch extension items also appear with `type: "Extension"` and
+`kind: "web.search"`; only that explicit kind maps to `WebSearch`.
+Repeated status updates and mirrored records collapse by their stable item id
+(or `call_id` for tool search); records without one remain distinct because
+query or result text is not a safe identity.
+
+The first observed status records activity, whether partial or completed. Native
+items add tool attribution only: they do not change token usage,
+`webSearchRequests`, or cost. Fork replay suppression compares native item IDs
+against the readable parent's native identities, so a new item can count before
+the usage replay anchor while an exact parent item stays suppressed. Items
+without IDs or without a readable parent retain burst suppression. When a
+native tool item arrives
+after the final usage record in its task, it is attached to that task's last
+call. Large image results and search result arrays stay out of the compact
+decoded entry; the decoder scans direct id/type/kind fields and copies only
+bounded field windows.
+
 ## Quirks
 
 - Newer Codex builds emit one `token_usage_record` per model response with a `response_id`. Those records are the preferred usage source from the first usable record onward; earlier `token_count` events in a mixed rollout still count, and malformed records leave the legacy fallback active.

src/codex-cache.ts

@@ -68,7 +68,9 @@ import { isWslUncPath } from './wsl.js'
 // GPT-5.3 Codex Spark instead of $0. Stored costUSD and speed are the old ones.
 // v22: a fork's replay burst drops only records found in the parent rollout
 // and counts the rest. v21 entries dropped the whole burst.
-export const CODEX_CACHE_VERSION = 22
+// v23: attribute the Responses built-in tools and item-model WebSearch records;
+// v22 exact entries have no such tool activity, so they must reparse.
+export const CODEX_CACHE_VERSION = 23
 export const CODEX_LEGACY_CACHE_FILE = 'codex-results.json'
 export function codexCacheFileName(version = CODEX_CACHE_VERSION): string {
   return `codex-results.v${version}.json`

src/codex-fork-replay.ts

@@ -2,6 +2,7 @@ import { stat } from 'fs/promises'
 import { basename, dirname } from 'path'
 
 import { readSessionLines } from './fs-utils.js'
+import { findRawObjectField, getRawJsonStringFieldAtObject } from './codex-jsonl-fields.js'
 import { walkRolloutFiles } from './launcher-homes.js'
 
 /** Maximum timestamp gap between copied records in Codex's fork replay burst. */
@@ -83,6 +84,7 @@ export function codexReplayUsageIdentity(info: { total_token_usage?: CodexUsageC
 }
 
 export const codexReplayResponseIdentity = (responseId: string): string => `response:${responseId}`
+export const codexReplayNativeToolIdentity = (tool: string, itemId: string): string => `native:${tool}:${itemId}`
 
 // Rollout names are `rollout-YYYY-MM-DDTHH-MM-SS-<id>.jsonl`.
 const ROLLOUT_ID_OFFSET = 'rollout-YYYY-MM-DDTHH-MM-SS-'.length
@@ -107,26 +109,94 @@ function codexHomeOf(path: string): string | undefined {
   return undefined
 }
 
-type ParentUsage = { sig: string; records: Promise<Array<[number, string]>> }
-const parentUsage = new Map<string, ParentUsage>()
+type ParentReplayRecords = { sig: string; records: Promise<Array<[number, string]>> }
+const parentReplayRecords = new Map<string, ParentReplayRecords>()
 
-async function readParentUsage(path: string): Promise<Array<[number, string]>> {
+function nativeToolForResponseType(type: string | undefined): 'WebSearch' | 'ToolSearch' | 'ImageGeneration' | undefined {
+  if (type === 'web_search_call') return 'WebSearch'
+  if (type === 'tool_search_call') return 'ToolSearch'
+  if (type === 'image_generation_call') return 'ImageGeneration'
+  return undefined
+}
+
+function addBufferParentReplayRecord(records: Array<[number, string]>, source: Buffer): void {
+  const entryType = getRawJsonStringFieldAtObject(source, 0, 'type')
+  if (!entryType) return
+  const timestamp = getRawJsonStringFieldAtObject(source, 0, 'timestamp')
+  const at = timestamp ? Date.parse(timestamp) : NaN
+  const payload = findRawObjectField(source, 0, 'payload')
+  if (!payload || source[payload.valueStart] !== 0x7b) return
+  const payloadStart = payload.valueStart
+  const payloadType = getRawJsonStringFieldAtObject(source, payloadStart, 'type')
+
+  // Keep the original usage identity semantics exactly: historical counters
+  // still pass through JSON.parse, including when a large line is a Buffer.
+  if (entryType === 'token_usage_record' || (entryType === 'event_msg' && payloadType === 'token_count')) {
+    addParsedParentReplayRecord(records, source.toString('utf-8'))
+    return
+  }
+
+  if (entryType === 'response_item') {
+    const tool = nativeToolForResponseType(payloadType)
+    if (!tool) return
+    const itemId = getRawJsonStringFieldAtObject(source, payloadStart, 'id')
+      ?? getRawJsonStringFieldAtObject(source, payloadStart, 'call_id')
+    if (itemId) records.push([at, codexReplayNativeToolIdentity(tool, itemId)])
+    return
+  }
+
+  if (entryType !== 'event_msg' || payloadType !== 'item_completed') return
+  const item = findRawObjectField(source, payloadStart, 'item')
+  if (!item || source[item.valueStart] !== 0x7b) return
+  const itemType = getRawJsonStringFieldAtObject(source, item.valueStart, 'type')
+  const itemKind = getRawJsonStringFieldAtObject(source, item.valueStart, 'kind')
+  if (itemType !== 'WebSearch' && !(itemType === 'Extension' && itemKind === 'web.search')) return
+  const itemId = getRawJsonStringFieldAtObject(source, item.valueStart, 'id')
+  if (itemId) records.push([at, codexReplayNativeToolIdentity('WebSearch', itemId)])
+}
+
+function addParsedParentReplayRecord(records: Array<[number, string]>, line: string): void {
+  let entry: { type?: string; timestamp?: string; payload?: { type?: string; response_id?: unknown; id?: unknown; call_id?: unknown; info?: Parameters<typeof codexReplayUsageIdentity>[0]; item?: { type?: unknown; id?: unknown; kind?: unknown } } }
+  try { entry = JSON.parse(line) } catch { return }
+  const at = entry.timestamp ? Date.parse(entry.timestamp) : NaN
+  if (entry.type === 'token_usage_record' && typeof entry.payload?.response_id === 'string') {
+    records.push([at, codexReplayResponseIdentity(entry.payload.response_id)])
+  } else if (entry.type === 'event_msg' && entry.payload?.type === 'token_count' && entry.payload.info) {
+    records.push([at, codexReplayUsageIdentity(entry.payload.info)])
+  } else if (entry.type === 'response_item') {
+    const tool = nativeToolForResponseType(entry.payload?.type)
+    const itemId = typeof entry.payload?.id === 'string' && entry.payload.id
+      ? entry.payload.id
+      : typeof entry.payload?.call_id === 'string' && entry.payload.call_id
+        ? entry.payload.call_id
+        : undefined
+    if (tool && itemId) records.push([at, codexReplayNativeToolIdentity(tool, itemId)])
+  } else if (entry.type === 'event_msg' && entry.payload?.type === 'item_completed'
+    && (entry.payload.item?.type === 'WebSearch'
+      || (entry.payload.item?.type === 'Extension' && entry.payload.item.kind === 'web.search'))
+    && typeof entry.payload.item.id === 'string' && entry.payload.item.id) {
+    records.push([at, codexReplayNativeToolIdentity('WebSearch', entry.payload.item.id)])
+  }
+}
+
+async function readParentReplayRecords(path: string): Promise<Array<[number, string]>> {
   const records: Array<[number, string]> = []
-  const skip = (head: string) => !head.includes('"token_count"') && !head.includes('"token_usage_record"')
-  for await (const line of readSessionLines(path, skip)) {
-    let entry: { type?: string; timestamp?: string; payload?: { type?: string; response_id?: unknown; info?: Parameters<typeof codexReplayUsageIdentity>[0] } }
-    try { entry = JSON.parse(line) } catch { continue }
-    const at = entry.timestamp ? Date.parse(entry.timestamp) : NaN
-    if (entry.type === 'token_usage_record' && typeof entry.payload?.response_id === 'string') {
-      records.push([at, codexReplayResponseIdentity(entry.payload.response_id)])
-    } else if (entry.type === 'event_msg' && entry.payload?.type === 'token_count' && entry.payload.info) {
-      records.push([at, codexReplayUsageIdentity(entry.payload.info)])
-    }
+  const skip = (head: string) => !head.includes('"token_count"')
+    && !head.includes('"token_usage_record"')
+    && !head.includes('"web_search_call"')
+    && !head.includes('"tool_search_call"')
+    && !head.includes('"image_generation_call"')
+    && !head.includes('"item_completed"')
+  for await (const line of readSessionLines(path, skip, { largeLineAsBuffer: true })) {
+    if (Buffer.isBuffer(line)) addBufferParentReplayRecord(records, line)
+    else addParsedParentReplayRecord(records, line)
   }
   return records
 }
 
-type ParentReplayLoader = (forkPath: string, parentId: string, forkedAtMs: number) => Promise<Set<string> | null>
+export type CodexParentReplayIndex = { usageIdentities: Set<string>; nativeIdentities: Set<string> }
+
+type ParentReplayLoader = (forkPath: string, parentId: string, forkedAtMs: number) => Promise<CodexParentReplayIndex | null>
 let parentReplayRequest: ParentReplayLoader | null = null
 
 /** A parse worker hands the lookup to the parent thread, so each parent is indexed and read once, not once per worker. */
@@ -135,12 +205,12 @@ export function routeCodexParentReplay(request: ParentReplayLoader): void {
 }
 
 /**
- * Usage identities the parent rollout recorded up to the fork, read from the
- * parent's raw file (its own copied history included, so a chain resolves one
- * level at a time). Null when the parent is not on disk next to the fork; the
- * caller then drops the whole replay burst.
+ * Usage and native item identities the parent rollout recorded up to the fork,
+ * read from the parent's raw file (its own copied history included, so a chain
+ * resolves one level at a time). A null result means the parent is not readable
+ * next to the fork; an empty native set means it was read and contains no items.
  */
-export async function loadCodexParentReplay(forkPath: string, parentId: string, forkedAtMs: number): Promise<Set<string> | null> {
+export async function loadCodexParentReplayIndex(forkPath: string, parentId: string, forkedAtMs: number): Promise<CodexParentReplayIndex | null> {
   if (parentReplayRequest) return parentReplayRequest(forkPath, parentId, forkedAtMs)
   const home = codexHomeOf(forkPath)
   const path = home ? findRollout(home, parentId) : undefined
@@ -152,12 +222,21 @@ export async function loadCodexParentReplay(forkPath: string, parentId: string,
   } catch {
     return null
   }
-  let cached = parentUsage.get(path)
+  let cached = parentReplayRecords.get(path)
   if (cached?.sig !== sig) {
-    cached = { sig, records: readParentUsage(path) }
-    parentUsage.set(path, cached)
+    cached = { sig, records: readParentReplayRecords(path) }
+    parentReplayRecords.set(path, cached)
   }
   // Records the parent wrote after the fork were never copied into it.
-  const ids = new Set((await cached.records).filter(([at]) => !(at > forkedAtMs)).map(([, id]) => id))
-  return ids.size > 0 ? ids : null
+  const records = (await cached.records).filter(([at]) => !(at > forkedAtMs))
+  return {
+    usageIdentities: new Set(records.map(([, id]) => id).filter(id => !id.startsWith('native:'))),
+    nativeIdentities: new Set(records.map(([, id]) => id).filter(id => id.startsWith('native:'))),
+  }
+}
+
+/** Usage-only compatibility wrapper; an empty identity set still means no replay anchor. */
+export async function loadCodexParentReplay(forkPath: string, parentId: string, forkedAtMs: number): Promise<Set<string> | null> {
+  const index = await loadCodexParentReplayIndex(forkPath, parentId, forkedAtMs)
+  return index?.usageIdentities.size ? index.usageIdentities : null
 }

src/codex-jsonl-fields.ts

@@ -0,0 +1,83 @@
+export type RawJsonFieldBounds = { keyStart: number; valueStart: number }
+
+/** Find a direct property in a JSON object without decoding its values. */
+export function findRawObjectField(source: Buffer, objectStart: number, field: string): RawJsonFieldBounds | undefined {
+  const fieldBytes = Buffer.from(field, 'utf-8')
+  let depth = 0
+  let rootStarted = false
+  let inString = false
+  let escaped = false
+  for (let i = objectStart; i < source.length; i++) {
+    const byte = source[i]!
+    if (!rootStarted) {
+      if (byte === 0x20 || byte === 0x09 || byte === 0x0a || byte === 0x0d) continue
+      if (byte !== 0x7b) return undefined // {
+      rootStarted = true
+      depth = 1
+      continue
+    }
+    if (inString) {
+      if (escaped) escaped = false
+      else if (byte === 0x5c) escaped = true // \\
+      else if (byte === 0x22) inString = false // "
+      continue
+    }
+    if (byte === 0x22) {
+      const keyStart = i + 1
+      let keyEnd = keyStart
+      let keyEscaped = false
+      for (; keyEnd < source.length; keyEnd++) {
+        const keyByte = source[keyEnd]!
+        if (keyEscaped) { keyEscaped = false; continue }
+        if (keyByte === 0x5c) { keyEscaped = true; continue }
+        if (keyByte === 0x22) break
+      }
+      if (depth === 1 && keyEnd < source.length) {
+        // Most strings at depth 1 are values, including potentially huge
+        // image results. Check the key/value delimiter and exact short-key
+        // length before comparing bytes so scanning never materializes them.
+        let colon = keyEnd + 1
+        while (colon < source.length && (source[colon] === 0x20 || source[colon] === 0x09 || source[colon] === 0x0a || source[colon] === 0x0d)) colon++
+        if (source[colon] === 0x3a && keyEnd - keyStart === fieldBytes.length && source.subarray(keyStart, keyEnd).equals(fieldBytes)) {
+          let valueStart = colon + 1
+          while (valueStart < source.length && (source[valueStart] === 0x20 || source[valueStart] === 0x09 || source[valueStart] === 0x0a || source[valueStart] === 0x0d)) valueStart++
+          return { keyStart: i, valueStart }
+        }
+      }
+      i = keyEnd
+      inString = false
+      continue
+    }
+    if (byte === 0x7b || byte === 0x5b) depth++ // { or [
+    else if (byte === 0x7d || byte === 0x5d) depth-- // } or ]
+    if (depth <= 0) break
+  }
+  return undefined
+}
+
+export function getRawObjectFieldWindow(source: Buffer, objectStart: number, field: string, windowBytes = 4096): string | undefined {
+  const bounds = findRawObjectField(source, objectStart, field)
+  return bounds ? source.subarray(bounds.keyStart, Math.min(source.length, bounds.keyStart + windowBytes)).toString('utf-8') : undefined
+}
+
+/** Read a short direct string property while bounding how much is decoded. */
+export function getRawJsonStringFieldAtObject(source: Buffer, objectStart: number, field: string, maxBytes = 4096): string | undefined {
+  const bounds = findRawObjectField(source, objectStart, field)
+  if (!bounds || source[bounds.valueStart] !== 0x22) return undefined
+
+  let escaped = false
+  const valueEndLimit = Math.min(source.length, bounds.valueStart + maxBytes + 1)
+  for (let i = bounds.valueStart + 1; i < valueEndLimit; i++) {
+    const byte = source[i]!
+    if (escaped) escaped = false
+    else if (byte === 0x5c) escaped = true
+    else if (byte === 0x22) {
+      try {
+        return JSON.parse(source.subarray(bounds.valueStart, i + 1).toString('utf-8')) as string
+      } catch {
+        return undefined
+      }
+    }
+  }
+  return undefined
+}

src/daily-cache.ts

@@ -361,7 +361,10 @@ import type { DateRange, ProjectSummary } from './types.js'
 // copilotCredits, and a chat session that also ran on the agent host is left
 // to it. That twin drops calls, so copilot joins
 // PENDING_REDERIVE_PROVIDER_VERSIONS at 75.
-export const DAILY_CACHE_VERSION = 75
+// v76: native Codex tool events now inform turn classification. Re-derive
+// settled days so their category totals reflect the recovered tool activity;
+// calls, tokens, and cost are unchanged.
+export const DAILY_CACHE_VERSION = 76
 const MIN_SUPPORTED_VERSION = 28
 
 /// Providers whose per-day CALL COUNT means something different at

src/parse-worker.ts

@@ -1,5 +1,5 @@
 import { parentPort, workerData, type MessagePort } from 'worker_threads'
-import { routeCodexParentReplay } from './codex-fork-replay.js'
+import { routeCodexParentReplay, type CodexParentReplayIndex } from './codex-fork-replay.js'
 import { restorePricingState, type PricingSnapshot } from './models.js'
 import type { ParentReplayReply, ParentReplayRequest, ParseJob } from './parse-workers.js'
 import { parseClaudeFileFull } from './parser.js'
@@ -11,13 +11,13 @@ if (!port) throw new Error('parse-worker must be started as a worker thread')
 const { pricing, replayPort } = workerData as { pricing: PricingSnapshot; replayPort: MessagePort }
 restorePricingState(pricing)
 
-const replayReplies = new Map<number, { resolve: (ids: Set<string> | null) => void; reject: (err: Error) => void }>()
+const replayReplies = new Map<number, { resolve: (index: CodexParentReplayIndex | null) => void; reject: (err: Error) => void }>()
 let nextReplayRequest = 0
 replayPort.on('message', (msg: ParentReplayReply) => {
   const reply = replayReplies.get(msg.id)
   replayReplies.delete(msg.id)
   if (msg.error !== undefined) reply?.reject(new Error(msg.error))
-  else reply?.resolve(msg.ids ?? null)
+  else reply?.resolve(msg.index ?? null)
 })
 routeCodexParentReplay((...args) => new Promise((resolve, reject) => {
   const id = nextReplayRequest++

src/parse-workers.ts

@@ -1,6 +1,6 @@
 import { availableParallelism, totalmem } from 'os'
 import { MessageChannel, Worker, type MessagePort } from 'worker_threads'
-import { loadCodexParentReplay } from './codex-fork-replay.js'
+import { loadCodexParentReplayIndex, type CodexParentReplayIndex } from './codex-fork-replay.js'
 import { snapshotPricingState } from './models.js'
 import type { ClaudeFileParse } from './parser.js'
 import type { SessionSource } from './providers/types.js'
@@ -124,13 +124,13 @@ type Task = { job: ParseJob; resolve: (r: ParseWorkerResult<unknown>) => void }
 
 type WorkerMessage = { json?: string | null; error?: string }
 
-export type ParentReplayRequest = { id: number; args: Parameters<typeof loadCodexParentReplay> }
-export type ParentReplayReply = { id: number; ids?: Set<string> | null; error?: string }
+export type ParentReplayRequest = { id: number; args: Parameters<typeof loadCodexParentReplayIndex> }
+export type ParentReplayReply = { id: number; index?: CodexParentReplayIndex | null; error?: string }
 
 function serveParentReplay(port: MessagePort): void {
   port.on('message', ({ id, args }: ParentReplayRequest) => {
-    loadCodexParentReplay(...args).then(
-      ids => port.postMessage({ id, ids } satisfies ParentReplayReply),
+    loadCodexParentReplayIndex(...args).then(
+      index => port.postMessage({ id, index } satisfies ParentReplayReply),
       (err: unknown) => port.postMessage({ id, error: err instanceof Error ? err.message : String(err) } satisfies ParentReplayReply),
     )
   })