diff --git a/apps/server/src/usage/UsageService.test.ts b/apps/server/src/usage/UsageService.test.ts index 0a9b9611eea0..2b8b6678dbdb 100644 --- a/apps/server/src/usage/UsageService.test.ts +++ b/apps/server/src/usage/UsageService.test.ts @@ -793,11 +793,12 @@ describe("UsageService", () => { }), ); - it.effect( - "repairs initial Codex prices after unchanged-file retries, persists evidence and corrections, and honors later native tiers", - () => + it.effect.each([false, true])( + "repairs initial Codex prices after unchanged-file retries, persists evidence and corrections, and honors later native tiers (historical: %s)", + (historical) => Effect.gen(function* () { const { home, settings } = yield* setup; + if (historical) yield* TestClock.setTime(Date.parse("2027-02-01T12:00:00Z")); const sessions = NodePath.join(home, "codex", "sessions"); const fastPath = NodePath.join(sessions, "fast.jsonl"); const ultraPath = NodePath.join(sessions, "ultra.jsonl"); @@ -842,6 +843,15 @@ describe("UsageService", () => { lines([meta("ultra-session"), context("ultra-turn"), usage(4)]), ); }); + if (historical) { + yield* Effect.promise(() => + Promise.all( + [fastPath, ultraPath].map((path) => + NodeFSP.utimes(path, CURSOR_NOW / 1000, CURSOR_NOW / 1000), + ), + ), + ); + } yield* Effect.gen(function* () { const first = yield* UsageService.make; verify(yield* first.readSummary(WINDOW), 7, 7); @@ -863,6 +873,11 @@ describe("UsageService", () => { yield* sql`ALTER TABLE unavailable_events RENAME TO orchestration_events`; // History repair must touch both the committed record and tentative tail. verify(yield* retry.readSummary(WINDOW), 7, 30); + if (historical) { + const config = yield* ServerSettings.ServerSettingsService; + yield* config.updateSettings({ usageHistoryRetentionDays: "unlimited" }); + verify(yield* retry.readSummary(WINDOW), 7, 30); + } yield* retry.awaitPersisted; yield* sql`ALTER TABLE orchestration_events RENAME TO resolved_events`; verify(yield* retry.readSummary(WINDOW), 7, 30); @@ -3746,53 +3761,85 @@ describe("UsageService", () => { }), ); - it.effect("keeps retained history when an obsolete scan finishes after retention widens", () => - Effect.gen(function* () { - const { transcript, settings, home } = yield* setup; - yield* TestClock.setTime(CURSOR_NOW); - const timestamp = CURSOR_NOW - 180 * 24 * HOUR_MS; - yield* Effect.promise(async () => { - await NodeFSP.writeFile( - transcript, - claudeLine(1, 11).replace( - "2026-08-01T10:00:00Z", - DateTime.formatIso(DateTime.makeUnsafe(timestamp)), - ), - ); - await NodeFSP.utimes(transcript, timestamp / 1000, timestamp / 1000); - }); - const started = yield* Deferred.make(); - const release = yield* Deferred.make(); - let scans = 0; - yield* Effect.gen(function* () { - const config = yield* ServerSettings.ServerSettingsService; - const service = yield* UsageService.make.pipe( - Effect.provideService(CursorUsageAccounts.CursorUsageAccounts, { - scan: () => - Effect.gen(function* () { - if (++scans === 1) { - yield* Deferred.succeed(started, undefined); - yield* Deferred.await(release); - } - return []; - }), - awaitPersisted: Effect.void, - }), - ); - const window = { ...WINDOW, sinceDay: UsageDay.make("2024-01-01") }; - const oldScan = yield* service.readSummary(window).pipe(Effect.forkChild); - yield* Deferred.await(started); - yield* config.updateSettings({ usageHistoryRetentionDays: "unlimited" }); - assert.strictEqual(totalOutputTokens(yield* service.readSummary(window)), 11); - yield* service.awaitPersisted; - yield* Effect.promise(() => NodeFSP.rm(transcript)); - yield* Deferred.succeed(release, undefined); - yield* Fiber.join(oldScan); - yield* service.awaitPersisted; - const restarted = yield* UsageService.make; - assert.strictEqual(totalOutputTokens(yield* restarted.readSummary(window)), 11); - }).pipe(Effect.provide(layerService({ prefix: "usage-retention-race", home, settings }))); - }), + it.effect.each([false, true])( + "keeps retained history when an obsolete scan finishes after retention widens (cold overlap: %s)", + (coldOverlap) => + Effect.gen(function* () { + const { transcript, settings, home } = yield* setup; + yield* TestClock.setTime(CURSOR_NOW); + const timestamp = CURSOR_NOW - 180 * 24 * HOUR_MS; + yield* Effect.promise(async () => { + await NodeFSP.writeFile( + transcript, + claudeLine(1, 11).replace( + "2026-08-01T10:00:00Z", + DateTime.formatIso(DateTime.makeUnsafe(timestamp)), + ), + ); + await NodeFSP.utimes(transcript, timestamp / 1000, timestamp / 1000); + }); + const started = yield* Deferred.make(); + const release = yield* Deferred.make(); + let scans = 0; + yield* Effect.gen(function* () { + const realFs = yield* Effect.promise(() => + vi.importActual("node:fs/promises"), + ); + const originalOpen = realFs.open; + const firstOpened = Promise.withResolvers(); + const secondOpened = Promise.withResolvers(); + const releaseFirst = Promise.withResolvers(); + const releaseSecond = Promise.withResolvers(); + let opens = 0; + using _open = vi.spyOn(NodeFSP, "open").mockImplementation(async (...args) => { + if (coldOverlap && args[0] === transcript) { + if (++opens === 1) { + firstOpened.resolve(); + await releaseFirst.promise; + } else if (opens === 2) { + secondOpened.resolve(); + await releaseSecond.promise; + } + } + return originalOpen(...args); + }); + const config = yield* ServerSettings.ServerSettingsService; + const service = yield* UsageService.make.pipe( + Effect.provideService(CursorUsageAccounts.CursorUsageAccounts, { + scan: () => + Effect.gen(function* () { + if (++scans === 1) { + yield* Deferred.succeed(started, undefined); + yield* Deferred.await(release); + } + return []; + }), + awaitPersisted: Effect.void, + }), + ); + const window = { ...WINDOW, sinceDay: UsageDay.make("2024-01-01") }; + const oldScan = yield* service.readSummary(window).pipe(Effect.forkChild); + yield* Deferred.await(started); + if (coldOverlap) yield* Effect.promise(() => firstOpened.promise); + yield* config.updateSettings({ usageHistoryRetentionDays: "unlimited" }); + const newScan = yield* service.readSummary(window).pipe(Effect.forkChild); + if (coldOverlap) { + yield* Effect.promise(() => secondOpened.promise); + releaseFirst.resolve(); + yield* Deferred.succeed(release, undefined); + yield* Fiber.join(oldScan); + releaseSecond.resolve(); + } + assert.strictEqual(totalOutputTokens(yield* Fiber.join(newScan)), 11); + yield* service.awaitPersisted; + yield* Effect.promise(() => NodeFSP.rm(transcript)); + yield* Deferred.succeed(release, undefined); + yield* Fiber.join(oldScan); + yield* service.awaitPersisted; + const restarted = yield* UsageService.make; + assert.strictEqual(totalOutputTokens(yield* restarted.readSummary(window)), 11); + }).pipe(Effect.provide(layerService({ prefix: "usage-retention-race", home, settings }))); + }), ); it.effect("credits the same copy of a duplicate after its transcripts are deleted", () => @@ -3967,6 +4014,96 @@ describe("UsageService", () => { }), ); + it.effect( + "reuses old live transcripts across all-time refreshes without retaining deleted history", + () => + Effect.gen(function* () { + yield* TestClock.setTime(Date.parse("2026-12-01T00:00:00Z")); + const { transcript, settings, home } = yield* setup; + const lastWrite = Date.parse("2026-08-01T10:00:00Z") / 1000; + yield* Effect.promise(async () => { + await NodeFSP.writeFile(transcript, claudeLine(1, 5)); + await NodeFSP.utimes(transcript, lastWrite, lastWrite); + }); + yield* Effect.gen(function* () { + const service = yield* UsageService.make; + const allTime = { + ...WINDOW, + sinceDay: UsageDay.make("1970-01-01"), + untilDay: UsageDay.make("2026-12-01"), + }; + const open = vi.spyOn(NodeFSP, "open"); + yield* Effect.addFinalizer(() => Effect.sync(() => open.mockRestore())); + open.mockClear(); + assert.strictEqual(totalOutputTokens(yield* service.readSummary(allTime)), 5); + assert.isTrue(open.mock.calls.some(([path]) => path === transcript)); + open.mockClear(); + for (const sinceDay of ["2026-11-02", "2026-09-03"]) { + const recent = { ...allTime, sinceDay: UsageDay.make(sinceDay) }; + assert.deepStrictEqual((yield* service.readSummary(recent)).buckets, []); + } + assert.strictEqual( + totalOutputTokens(yield* service.readSummary({ ...allTime, awaitRefresh: true })), + 5, + ); + assert.isFalse(open.mock.calls.some(([path]) => path === transcript)); + + yield* Effect.promise(async () => { + await NodeFSP.appendFile(transcript, claudeLine(2, 7)); + await NodeFSP.utimes(transcript, lastWrite + 86400, lastWrite + 86400); + }); + assert.strictEqual(totalOutputTokens(yield* service.readSummary(allTime)), 12); + yield* Effect.promise(async () => { + await NodeFSP.writeFile(transcript, claudeLine(3, 9)); + await NodeFSP.utimes(transcript, lastWrite + 172800, lastWrite + 172800); + }); + assert.strictEqual(totalOutputTokens(yield* service.readSummary(allTime)), 9); + + yield* Effect.promise(() => NodeFSP.rm(transcript)); + assert.deepStrictEqual((yield* service.readSummary(allTime)).buckets, []); + yield* service.awaitPersisted; + const restarted = yield* UsageService.make; + assert.deepStrictEqual((yield* restarted.readSummary(allTime)).buckets, []); + }).pipe( + Effect.provide(layerService({ prefix: "usage-service-all-time-test", home, settings })), + ); + }).pipe(Effect.scoped), + ); + + it.effect("answers a provisional all-time read with old and recent transcripts", () => + Effect.gen(function* () { + yield* TestClock.setTime(Date.parse("2026-12-01T00:00:00Z")); + const { transcript, settings, home } = yield* setup; + const recent = NodePath.join(NodePath.dirname(transcript), "recent.jsonl"); + yield* Effect.promise(async () => { + const oldWrite = Date.parse("2026-08-01T10:00:00Z") / 1000; + const recentWrite = Date.parse("2026-11-30T10:00:00Z") / 1000; + await NodeFSP.writeFile(transcript, claudeLine(1, 5)); + await NodeFSP.utimes(transcript, oldWrite, oldWrite); + await NodeFSP.writeFile(recent, claudeLine(2, 3)); + await NodeFSP.utimes(recent, recentWrite, recentWrite); + }); + yield* Effect.gen(function* () { + const service = yield* UsageService.make; + const allTime = { + ...WINDOW, + sinceDay: UsageDay.make("1970-01-01"), + untilDay: UsageDay.make("2026-12-01"), + }; + assert.strictEqual(totalOutputTokens(yield* service.readSummary(allTime)), 8); + // The recent file keeps the source refreshing, so this read cannot fall back to an exact scan. + assert.strictEqual( + totalOutputTokens(yield* service.readSummary({ ...allTime, provisional: true })), + 8, + ); + }).pipe( + Effect.provide( + layerService({ prefix: "usage-service-all-time-provisional", home, settings }), + ), + ); + }).pipe(Effect.scoped), + ); + it.effect("reports saved usage of a removed directory only for windows it reaches", () => Effect.gen(function* () { yield* TestClock.setTime(Date.parse("2026-08-12T00:00:00Z")); diff --git a/apps/server/src/usage/UsageService.ts b/apps/server/src/usage/UsageService.ts index 30edc903a580..4b8cde138035 100644 --- a/apps/server/src/usage/UsageService.ts +++ b/apps/server/src/usage/UsageService.ts @@ -193,6 +193,8 @@ export const make = Effect.gen(function* () { const readerContext = yield* Effect.context(); const fileCache: ScanCache = new Map(); + // Old live files stay warm without enlarging persisted history or normal-window cache walks. + const historicalFileCache: ScanCache = new Map(); // Reads from concurrent windows can still hold the original record after it is corrected. const resolvedRecords = new WeakMap(); const correctedRecord = (record: UsageRecord) => resolvedRecords.get(record) ?? record; @@ -568,7 +570,7 @@ export const make = Effect.gen(function* () { readonly failed?: true; readonly update?: { readonly entry: CachedFile; readonly replaces: CachedFile | undefined }; }> { - const cached = fileCache.get(filePath); + const cached = fileCache.get(filePath) ?? historicalFileCache.get(filePath); // Provider is part of the identity: if both providers were ever pointed // at one directory, a hit parsed by the other parser must not be reused. if ( @@ -698,6 +700,7 @@ export const make = Effect.gen(function* () { const scanTranscriptDir = Effect.fn("UsageService.scanTranscriptDir")(function* ( source: TranscriptSource, windowStartMs: number, + retentionCutoffMs: number, fromMemory: boolean, ) { const { provider, format, dir, volumeId, physicalSourceId, fileName } = source; @@ -719,6 +722,17 @@ export const make = Effect.gen(function* () { const { files, walked, failedPaths } = yield* Effect.promise(() => listTranscriptFiles(dir, windowStartMs, fileName === undefined ? undefined : { fileName }), ); + if (windowStartMs < retentionCutoffMs && failedPaths === 0) { + const livePaths = new Set(files.map((file) => file.path)); + for (const [filePath, entry] of historicalFileCache) { + if ( + entry.mtimeMs >= windowStartMs && + isWithinDirectory(filePath, dir) && + !livePaths.has(filePath) + ) + historicalFileCache.delete(filePath); + } + } const read = yield* Effect.forEach( files, (file) => @@ -728,21 +742,32 @@ export const make = Effect.gen(function* () { { concurrency: TRANSCRIPT_READ_CONCURRENCY }, ); const parsedFiles = read.map(({ path, records, malformedRecords, update }) => { - if (update === undefined) return { path, records, malformedRecords }; // A scan of another window may have cached its own read of this file // meanwhile. Then keep whichever read saw the later file, so a slower // scan never replaces newer usage with older. - const current = fileCache.get(path); + const current = fileCache.get(path) ?? historicalFileCache.get(path); if ( - current === update.replaces || - current === undefined || - isLaterRead(update.entry, current) + update !== undefined && + (current === update.replaces || current === undefined || isLaterRead(update.entry, current)) ) { - fileCache.set(path, { + const cache = update.entry.mtimeMs < retentionCutoffMs ? historicalFileCache : fileCache; + if (cache === fileCache) historicalFileCache.delete(path); + else if (fileCache.delete(path)) cacheDirty = true; + cache.set(path, { ...update.entry, records: correctedBatch(update.entry.records), tailRecords: correctedBatch(update.entry.tailRecords), }); + if (cache === fileCache) cacheDirty = true; + } + const historical = historicalFileCache.get(path); + if (historical !== undefined && historical.mtimeMs >= retentionCutoffMs) { + historicalFileCache.delete(path); + fileCache.set(path, { + ...historical, + records: correctedBatch(historical.records), + tailRecords: correctedBatch(historical.tailRecords), + }); cacheDirty = true; } return { path, records, malformedRecords }; @@ -815,7 +840,12 @@ export const make = Effect.gen(function* () { // Scan independent sources together, directories sequentially to bound open files; keep order, since the first copy decides a duplicate's bucket. const [transcripts, scanDirs] = yield* Effect.all( - [Effect.forEach(dirs, (dir) => scanTranscriptDir(dir, windowStartMs, fromMemory)), scans], + [ + Effect.forEach(dirs, (dir) => + scanTranscriptDir(dir, windowStartMs, retentionCutoffMs, fromMemory), + ), + scans, + ], { concurrency: "unbounded" }, ); const scanned: readonly ScannedDir[] = [...transcripts, ...pending, ...scanDirs.flat()]; @@ -883,8 +913,10 @@ export const make = Effect.gen(function* () { ); // A window no complete scan has read would show partial totals, so every directory must cover it. // Saved files are keyed by path, so two providers reading one directory keep only one parse. + // Windows reaching past retention need the in-memory historical files, which a saved snapshot lacks. const fromMemory = provisional && + windowStartMs >= retentionCutoffMs && resolved.dirs.length > 0 && new Set(resolved.dirs.map((source) => source.dir)).size === resolved.dirs.length && resolved.dirs.every( diff --git a/apps/web/src/components/settings/KeybindingsSettings.logic.ts b/apps/web/src/components/settings/KeybindingsSettings.logic.ts index 99ff61182d62..e89cc4379184 100644 --- a/apps/web/src/components/settings/KeybindingsSettings.logic.ts +++ b/apps/web/src/components/settings/KeybindingsSettings.logic.ts @@ -509,6 +509,7 @@ const USAGE_COMMAND_LABELS: Record< "usage.period.week": msg("Usage: Period: 7 days"), "usage.period.month": msg("Usage: Period: 30 days"), "usage.period.quarter": msg("Usage: Period: 90 days"), + "usage.period.all": msg("Usage: Period: All time"), }; const LATE_COMMAND_LABELS: Partial> = { diff --git a/apps/web/src/components/usage/UsagePage.tsx b/apps/web/src/components/usage/UsagePage.tsx index adb05afb92eb..c499fca9c4ae 100644 --- a/apps/web/src/components/usage/UsagePage.tsx +++ b/apps/web/src/components/usage/UsagePage.tsx @@ -46,12 +46,10 @@ import { enumerateHourStarts, formatCount, formatDateTimeShort, - formatDayShort, formatHourShort, formatPercent, formatTokens, formatUsd, - makeWindow, } from "@t3tools/shared/usageFormat"; import { formatUsageContractMismatch, usageSourceMessages } from "@t3tools/shared/usageFormatCopy"; import { Button, InlineButton } from "../ui/button"; @@ -79,7 +77,8 @@ import { import { WorkspacePageContainer } from "../WorkspacePageContainer"; import { UsageLimitsSection } from "./UsageLimits"; import { UsagePriceOverrides } from "./UsagePriceOverrides"; -import { UsageProviderChart } from "./UsageProviderChart"; +import { formatChartDay, UsageProviderChart } from "./UsageProviderChart"; +import { makeUsageWindow, trimUsageWindow } from "./usageWindow"; import { SpeedPremium, UsageModelDialog } from "./UsageModelDialog"; import { UsageShareBar } from "./UsageShareBar"; import { @@ -193,11 +192,7 @@ export function UsagePage() { }; const [windowSelection, setWindowSelection] = useState(() => ({ days: preferences.windowDays, - window: makeWindow( - preferences.windowDays, - undefined, - preferences.windowDays === 1 ? "hour" : "day", - ), + window: makeUsageWindow(preferences.windowDays), })); const metric = preferences.metric; const showingLimits = metric === "limits"; @@ -209,7 +204,6 @@ export function UsagePage() { useState | null>(null); const hiddenProviders = new Set(preferences.hiddenProviders); const { days: windowDays, window } = windowSelection; - const isPast24Hours = windowDays === 1; const { merged: answeredUsage, environments, @@ -219,7 +213,7 @@ export function UsagePage() { refresh, } = useUsage(window, selectedEnvironmentIds, hiddenProviders); const merged = shown?.merged ?? answeredUsage; - const shownWindow = shown?.window ?? window; + const shownWindow = trimUsageWindow(shown?.window ?? window, merged.daily); const shownHourly = shownWindow.resolution === "hour"; const refreshingUsage = isRefreshing && !showingLimits; const showingKept = shown !== null && shown.window !== window; @@ -245,7 +239,7 @@ export function UsagePage() { updatePreferences({ windowDays: days }); setWindowSelection({ days, - window: makeWindow(days, undefined, days === 1 ? "hour" : "day"), + window: makeUsageWindow(days), }); }; const selectMetric = (nextMetric: UsageMetric) => { @@ -310,7 +304,7 @@ export function UsagePage() { }); return; } - const nextWindow = makeWindow(windowDays, undefined, isPast24Hours ? "hour" : "day"); + const nextWindow = makeUsageWindow(windowDays); if ( nextWindow.sinceDay !== window.sinceDay || nextWindow.untilDay !== window.untilDay || @@ -357,6 +351,7 @@ export function UsagePage() { (presentation.serverConfig?.providers ?? []).some(usesChatGptSharing), ); + const shownWindowDays = [shownWindow.sinceDay, shownWindow.untilDay]; const windowLabel = shownHourly && shownWindow.sinceTime !== undefined && shownWindow.untilTime !== undefined ? gt("{start} to {end}", { @@ -364,13 +359,13 @@ export function UsagePage() { end: formatDateTimeShort(shownWindow.untilTime, shownWindow.timeZone, hour12, locale), }) : gt("{start} to {end}", { - start: formatDayShort(shownWindow.sinceDay, locale), - end: formatDayShort(shownWindow.untilDay, locale), + start: formatChartDay(shownWindow.sinceDay, shownWindowDays, locale), + end: formatChartDay(shownWindow.untilDay, shownWindowDays, locale), }); const windowOptionLabel = WINDOW_OPTIONS.find((option) => option.days === windowDays)?.label; const metricOptionLabel = METRIC_OPTIONS.find((option) => option.value === metric)?.label; const topbarContent = ( -
+

@@ -405,7 +400,7 @@ export function UsagePage() { {windowLabel} ) : null} -
+
{/* Limits are always current, so they have no period. The period sits before the metric tabs so dropping it never moves them. */} {showingLimits ? null : ( @@ -451,7 +446,7 @@ export function UsagePage() {
-
+
{showingLimits ? null : (