Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
237 changes: 187 additions & 50 deletions apps/server/src/usage/UsageService.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -793,11 +793,12 @@ describe("UsageService", () => {
}),
);

it.effect(
"repairs initial Codex prices after unchanged-file retries, persists evidence and corrections, and honors later native tiers",
() =>
it.effect.each([false, true])(
"repairs initial Codex prices after unchanged-file retries, persists evidence and corrections, and honors later native tiers (historical: %s)",
(historical) =>
Effect.gen(function* () {
const { home, settings } = yield* setup;
if (historical) yield* TestClock.setTime(Date.parse("2027-02-01T12:00:00Z"));
const sessions = NodePath.join(home, "codex", "sessions");
const fastPath = NodePath.join(sessions, "fast.jsonl");
const ultraPath = NodePath.join(sessions, "ultra.jsonl");
Expand Down Expand Up @@ -842,6 +843,15 @@ describe("UsageService", () => {
lines([meta("ultra-session"), context("ultra-turn"), usage(4)]),
);
});
if (historical) {
yield* Effect.promise(() =>
Promise.all(
[fastPath, ultraPath].map((path) =>
NodeFSP.utimes(path, CURSOR_NOW / 1000, CURSOR_NOW / 1000),
),
),
);
}
yield* Effect.gen(function* () {
const first = yield* UsageService.make;
verify(yield* first.readSummary(WINDOW), 7, 7);
Expand All @@ -863,6 +873,11 @@ describe("UsageService", () => {
yield* sql`ALTER TABLE unavailable_events RENAME TO orchestration_events`;
// History repair must touch both the committed record and tentative tail.
verify(yield* retry.readSummary(WINDOW), 7, 30);
if (historical) {
const config = yield* ServerSettings.ServerSettingsService;
yield* config.updateSettings({ usageHistoryRetentionDays: "unlimited" });
verify(yield* retry.readSummary(WINDOW), 7, 30);
}
yield* retry.awaitPersisted;
yield* sql`ALTER TABLE orchestration_events RENAME TO resolved_events`;
verify(yield* retry.readSummary(WINDOW), 7, 30);
Expand Down Expand Up @@ -3746,53 +3761,85 @@ describe("UsageService", () => {
}),
);

it.effect("keeps retained history when an obsolete scan finishes after retention widens", () =>
Effect.gen(function* () {
const { transcript, settings, home } = yield* setup;
yield* TestClock.setTime(CURSOR_NOW);
const timestamp = CURSOR_NOW - 180 * 24 * HOUR_MS;
yield* Effect.promise(async () => {
await NodeFSP.writeFile(
transcript,
claudeLine(1, 11).replace(
"2026-08-01T10:00:00Z",
DateTime.formatIso(DateTime.makeUnsafe(timestamp)),
),
);
await NodeFSP.utimes(transcript, timestamp / 1000, timestamp / 1000);
});
const started = yield* Deferred.make<void>();
const release = yield* Deferred.make<void>();
let scans = 0;
yield* Effect.gen(function* () {
const config = yield* ServerSettings.ServerSettingsService;
const service = yield* UsageService.make.pipe(
Effect.provideService(CursorUsageAccounts.CursorUsageAccounts, {
scan: () =>
Effect.gen(function* () {
if (++scans === 1) {
yield* Deferred.succeed(started, undefined);
yield* Deferred.await(release);
}
return [];
}),
awaitPersisted: Effect.void,
}),
);
const window = { ...WINDOW, sinceDay: UsageDay.make("2024-01-01") };
const oldScan = yield* service.readSummary(window).pipe(Effect.forkChild);
yield* Deferred.await(started);
yield* config.updateSettings({ usageHistoryRetentionDays: "unlimited" });
assert.strictEqual(totalOutputTokens(yield* service.readSummary(window)), 11);
yield* service.awaitPersisted;
yield* Effect.promise(() => NodeFSP.rm(transcript));
yield* Deferred.succeed(release, undefined);
yield* Fiber.join(oldScan);
yield* service.awaitPersisted;
const restarted = yield* UsageService.make;
assert.strictEqual(totalOutputTokens(yield* restarted.readSummary(window)), 11);
}).pipe(Effect.provide(layerService({ prefix: "usage-retention-race", home, settings })));
}),
it.effect.each([false, true])(
"keeps retained history when an obsolete scan finishes after retention widens (cold overlap: %s)",
(coldOverlap) =>
Effect.gen(function* () {
const { transcript, settings, home } = yield* setup;
yield* TestClock.setTime(CURSOR_NOW);
const timestamp = CURSOR_NOW - 180 * 24 * HOUR_MS;
yield* Effect.promise(async () => {
await NodeFSP.writeFile(
transcript,
claudeLine(1, 11).replace(
"2026-08-01T10:00:00Z",
DateTime.formatIso(DateTime.makeUnsafe(timestamp)),
),
);
await NodeFSP.utimes(transcript, timestamp / 1000, timestamp / 1000);
});
const started = yield* Deferred.make<void>();
const release = yield* Deferred.make<void>();
let scans = 0;
yield* Effect.gen(function* () {
const realFs = yield* Effect.promise(() =>
vi.importActual<typeof NodeFSP>("node:fs/promises"),
);
const originalOpen = realFs.open;
const firstOpened = Promise.withResolvers<void>();
const secondOpened = Promise.withResolvers<void>();
const releaseFirst = Promise.withResolvers<void>();
const releaseSecond = Promise.withResolvers<void>();
let opens = 0;
using _open = vi.spyOn(NodeFSP, "open").mockImplementation(async (...args) => {
if (coldOverlap && args[0] === transcript) {
if (++opens === 1) {
firstOpened.resolve();
await releaseFirst.promise;
} else if (opens === 2) {
secondOpened.resolve();
await releaseSecond.promise;
}
}
return originalOpen(...args);
});
const config = yield* ServerSettings.ServerSettingsService;
const service = yield* UsageService.make.pipe(
Effect.provideService(CursorUsageAccounts.CursorUsageAccounts, {
scan: () =>
Effect.gen(function* () {
if (++scans === 1) {
yield* Deferred.succeed(started, undefined);
yield* Deferred.await(release);
}
return [];
}),
awaitPersisted: Effect.void,
}),
);
const window = { ...WINDOW, sinceDay: UsageDay.make("2024-01-01") };
const oldScan = yield* service.readSummary(window).pipe(Effect.forkChild);
yield* Deferred.await(started);
if (coldOverlap) yield* Effect.promise(() => firstOpened.promise);
yield* config.updateSettings({ usageHistoryRetentionDays: "unlimited" });
const newScan = yield* service.readSummary(window).pipe(Effect.forkChild);
if (coldOverlap) {
yield* Effect.promise(() => secondOpened.promise);
releaseFirst.resolve();
yield* Deferred.succeed(release, undefined);
yield* Fiber.join(oldScan);
releaseSecond.resolve();
}
assert.strictEqual(totalOutputTokens(yield* Fiber.join(newScan)), 11);
yield* service.awaitPersisted;
yield* Effect.promise(() => NodeFSP.rm(transcript));
yield* Deferred.succeed(release, undefined);
yield* Fiber.join(oldScan);
yield* service.awaitPersisted;
const restarted = yield* UsageService.make;
assert.strictEqual(totalOutputTokens(yield* restarted.readSummary(window)), 11);
}).pipe(Effect.provide(layerService({ prefix: "usage-retention-race", home, settings })));
}),
);

it.effect("credits the same copy of a duplicate after its transcripts are deleted", () =>
Expand Down Expand Up @@ -3967,6 +4014,96 @@ describe("UsageService", () => {
}),
);

it.effect(
"reuses old live transcripts across all-time refreshes without retaining deleted history",
() =>
Effect.gen(function* () {
yield* TestClock.setTime(Date.parse("2026-12-01T00:00:00Z"));
const { transcript, settings, home } = yield* setup;
const lastWrite = Date.parse("2026-08-01T10:00:00Z") / 1000;
yield* Effect.promise(async () => {
await NodeFSP.writeFile(transcript, claudeLine(1, 5));
await NodeFSP.utimes(transcript, lastWrite, lastWrite);
});
yield* Effect.gen(function* () {
const service = yield* UsageService.make;
const allTime = {
...WINDOW,
sinceDay: UsageDay.make("1970-01-01"),
untilDay: UsageDay.make("2026-12-01"),
};
const open = vi.spyOn(NodeFSP, "open");
yield* Effect.addFinalizer(() => Effect.sync(() => open.mockRestore()));
open.mockClear();
assert.strictEqual(totalOutputTokens(yield* service.readSummary(allTime)), 5);
assert.isTrue(open.mock.calls.some(([path]) => path === transcript));
open.mockClear();
for (const sinceDay of ["2026-11-02", "2026-09-03"]) {
const recent = { ...allTime, sinceDay: UsageDay.make(sinceDay) };
assert.deepStrictEqual((yield* service.readSummary(recent)).buckets, []);
}
assert.strictEqual(
totalOutputTokens(yield* service.readSummary({ ...allTime, awaitRefresh: true })),
5,
);
assert.isFalse(open.mock.calls.some(([path]) => path === transcript));

yield* Effect.promise(async () => {
await NodeFSP.appendFile(transcript, claudeLine(2, 7));
await NodeFSP.utimes(transcript, lastWrite + 86400, lastWrite + 86400);
});
assert.strictEqual(totalOutputTokens(yield* service.readSummary(allTime)), 12);
yield* Effect.promise(async () => {
await NodeFSP.writeFile(transcript, claudeLine(3, 9));
await NodeFSP.utimes(transcript, lastWrite + 172800, lastWrite + 172800);
});
assert.strictEqual(totalOutputTokens(yield* service.readSummary(allTime)), 9);

yield* Effect.promise(() => NodeFSP.rm(transcript));
assert.deepStrictEqual((yield* service.readSummary(allTime)).buckets, []);
yield* service.awaitPersisted;
const restarted = yield* UsageService.make;
assert.deepStrictEqual((yield* restarted.readSummary(allTime)).buckets, []);
}).pipe(
Effect.provide(layerService({ prefix: "usage-service-all-time-test", home, settings })),
);
}).pipe(Effect.scoped),
);

it.effect("answers a provisional all-time read with old and recent transcripts", () =>
Effect.gen(function* () {
yield* TestClock.setTime(Date.parse("2026-12-01T00:00:00Z"));
const { transcript, settings, home } = yield* setup;
const recent = NodePath.join(NodePath.dirname(transcript), "recent.jsonl");
yield* Effect.promise(async () => {
const oldWrite = Date.parse("2026-08-01T10:00:00Z") / 1000;
const recentWrite = Date.parse("2026-11-30T10:00:00Z") / 1000;
await NodeFSP.writeFile(transcript, claudeLine(1, 5));
await NodeFSP.utimes(transcript, oldWrite, oldWrite);
await NodeFSP.writeFile(recent, claudeLine(2, 3));
await NodeFSP.utimes(recent, recentWrite, recentWrite);
});
yield* Effect.gen(function* () {
const service = yield* UsageService.make;
const allTime = {
...WINDOW,
sinceDay: UsageDay.make("1970-01-01"),
untilDay: UsageDay.make("2026-12-01"),
};
assert.strictEqual(totalOutputTokens(yield* service.readSummary(allTime)), 8);
// The recent file keeps the source refreshing, so this read cannot fall back to an exact scan.
assert.strictEqual(
totalOutputTokens(yield* service.readSummary({ ...allTime, provisional: true })),
8,
);
}).pipe(
Effect.provide(
layerService({ prefix: "usage-service-all-time-provisional", home, settings }),
),
);
}).pipe(Effect.scoped),
);

it.effect("reports saved usage of a removed directory only for windows it reaches", () =>
Effect.gen(function* () {
yield* TestClock.setTime(Date.parse("2026-08-12T00:00:00Z"));
Expand Down
48 changes: 40 additions & 8 deletions apps/server/src/usage/UsageService.ts
Original file line number Diff line number Diff line change
Expand Up @@ -193,6 +193,8 @@ export const make = Effect.gen(function* () {
const readerContext = yield* Effect.context<BuiltInUsageReadersEnv>();

const fileCache: ScanCache = new Map();
// Old live files stay warm without enlarging persisted history or normal-window cache walks.
const historicalFileCache: ScanCache = new Map();
// Reads from concurrent windows can still hold the original record after it is corrected.
const resolvedRecords = new WeakMap<UsageRecord, UsageRecord>();
const correctedRecord = (record: UsageRecord) => resolvedRecords.get(record) ?? record;
Expand Down Expand Up @@ -568,7 +570,7 @@ export const make = Effect.gen(function* () {
readonly failed?: true;
readonly update?: { readonly entry: CachedFile; readonly replaces: CachedFile | undefined };
}> {
const cached = fileCache.get(filePath);
const cached = fileCache.get(filePath) ?? historicalFileCache.get(filePath);
// Provider is part of the identity: if both providers were ever pointed
// at one directory, a hit parsed by the other parser must not be reused.
if (
Expand Down Expand Up @@ -698,6 +700,7 @@ export const make = Effect.gen(function* () {
const scanTranscriptDir = Effect.fn("UsageService.scanTranscriptDir")(function* (
source: TranscriptSource,
windowStartMs: number,
retentionCutoffMs: number,
fromMemory: boolean,
) {
const { provider, format, dir, volumeId, physicalSourceId, fileName } = source;
Expand All @@ -719,6 +722,17 @@ export const make = Effect.gen(function* () {
const { files, walked, failedPaths } = yield* Effect.promise(() =>
listTranscriptFiles(dir, windowStartMs, fileName === undefined ? undefined : { fileName }),
);
if (windowStartMs < retentionCutoffMs && failedPaths === 0) {
const livePaths = new Set(files.map((file) => file.path));
for (const [filePath, entry] of historicalFileCache) {
if (
entry.mtimeMs >= windowStartMs &&
isWithinDirectory(filePath, dir) &&
!livePaths.has(filePath)
)
historicalFileCache.delete(filePath);
}
}
const read = yield* Effect.forEach(
files,
(file) =>
Expand All @@ -728,21 +742,32 @@ export const make = Effect.gen(function* () {
{ concurrency: TRANSCRIPT_READ_CONCURRENCY },
);
const parsedFiles = read.map(({ path, records, malformedRecords, update }) => {
if (update === undefined) return { path, records, malformedRecords };
// A scan of another window may have cached its own read of this file
// meanwhile. Then keep whichever read saw the later file, so a slower
// scan never replaces newer usage with older.
const current = fileCache.get(path);
const current = fileCache.get(path) ?? historicalFileCache.get(path);
if (
current === update.replaces ||
current === undefined ||
isLaterRead(update.entry, current)
update !== undefined &&
(current === update.replaces || current === undefined || isLaterRead(update.entry, current))
) {
fileCache.set(path, {
const cache = update.entry.mtimeMs < retentionCutoffMs ? historicalFileCache : fileCache;
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Resolved
if (cache === fileCache) historicalFileCache.delete(path);
else if (fileCache.delete(path)) cacheDirty = true;
cache.set(path, {
...update.entry,
records: correctedBatch(update.entry.records),
tailRecords: correctedBatch(update.entry.tailRecords),
});
if (cache === fileCache) cacheDirty = true;
}
const historical = historicalFileCache.get(path);
if (historical !== undefined && historical.mtimeMs >= retentionCutoffMs) {
historicalFileCache.delete(path);
fileCache.set(path, {
...historical,
records: correctedBatch(historical.records),
tailRecords: correctedBatch(historical.tailRecords),
});
cacheDirty = true;
}
return { path, records, malformedRecords };
Expand Down Expand Up @@ -815,7 +840,12 @@ export const make = Effect.gen(function* () {

// Scan independent sources together, directories sequentially to bound open files; keep order, since the first copy decides a duplicate's bucket.
const [transcripts, scanDirs] = yield* Effect.all(
[Effect.forEach(dirs, (dir) => scanTranscriptDir(dir, windowStartMs, fromMemory)), scans],
[
Effect.forEach(dirs, (dir) =>
scanTranscriptDir(dir, windowStartMs, retentionCutoffMs, fromMemory),
),
scans,
],
{ concurrency: "unbounded" },
);
const scanned: readonly ScannedDir[] = [...transcripts, ...pending, ...scanDirs.flat()];
Expand Down Expand Up @@ -883,8 +913,10 @@ export const make = Effect.gen(function* () {
);
// A window no complete scan has read would show partial totals, so every directory must cover it.
// Saved files are keyed by path, so two providers reading one directory keep only one parse.
// Windows reaching past retention need the in-memory historical files, which a saved snapshot lacks.
const fromMemory =
provisional &&
windowStartMs >= retentionCutoffMs &&
resolved.dirs.length > 0 &&
new Set(resolved.dirs.map((source) => source.dir)).size === resolved.dirs.length &&
resolved.dirs.every(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -509,6 +509,7 @@ const USAGE_COMMAND_LABELS: Record<
"usage.period.week": msg("Usage: Period: 7 days"),
"usage.period.month": msg("Usage: Period: 30 days"),
"usage.period.quarter": msg("Usage: Period: 90 days"),
"usage.period.all": msg("Usage: Period: All time"),
};

const LATE_COMMAND_LABELS: Partial<Record<KeybindingCommand, string>> = {
Expand Down
Loading
Loading