Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
38 changes: 38 additions & 0 deletions apps/server/src/workspace/WorkspaceEntries.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import * as FileSystem from "effect/FileSystem";
import * as Layer from "effect/Layer";
import * as Path from "effect/Path";
import * as PlatformError from "effect/PlatformError";
import * as TestClock from "effect/testing/TestClock";
import { vi } from "vite-plus/test";

import * as ServerConfig from "../config.ts";
Expand Down Expand Up @@ -712,6 +713,43 @@ it.layer(layerTest, { excludeTestServices: true })("WorkspaceEntries", (it) => {
}
}),
);

it.effect("releases an index nobody searched within the idle timeout on refresh", () =>
Effect.gen(function* () {
const cwd = yield* makeTempDir({ prefix: "t3code-workspace-idle-refresh-" });
yield* writeTextFile(cwd, "src/index.ts");
let rescans = 0;
let destroyed = 0;
const createFinder = FileFinder.create.bind(FileFinder);
vi.spyOn(FileFinder, "create").mockImplementation((options) => {
const created = createFinder(options);
if (created.ok) {
const finder = created.value;
const scanFilesAsync = finder.scanFilesAsync.bind(finder);
finder.scanFilesAsync = () => {
rescans += 1;
return scanFilesAsync();
};
const destroy = finder.destroy.bind(finder);
finder.destroy = () => {
destroyed += 1;
destroy();
};
}
return created;
});
const workspaceEntries = yield* WorkspaceEntries.WorkspaceEntries;
yield* TestClock.adjust("1 hour");
yield* searchWorkspaceEntries({ cwd, query: "index", limit: 10 });

for (let run = 0; run < 3; run++) {
yield* TestClock.adjust("6 minutes");
yield* workspaceEntries.refresh(cwd);
}

expect({ rescans, destroyed }).toEqual({ rescans: 2, destroyed: 1 });
}).pipe(Effect.provide(TestClock.layer())),
);
});

describe("searchContents", () => {
Expand Down
50 changes: 29 additions & 21 deletions apps/server/src/workspace/WorkspaceEntries.ts
Original file line number Diff line number Diff line change
@@ -1,7 +1,9 @@
// @effect-diagnostics nodeBuiltinImport:off
import * as NodeFSP from "node:fs/promises";

import * as Clock from "effect/Clock";
import * as Context from "effect/Context";
import * as Duration from "effect/Duration";
import * as Effect from "effect/Effect";
import * as Layer from "effect/Layer";
import * as Path from "effect/Path";
Expand Down Expand Up @@ -165,6 +167,18 @@ export const make = Effect.gen(function* () {
return yield* workspacePaths.normalizeWorkspaceRoot(cwd);
});

// Refreshing goes through `get`, which restarts the map's idle timer, so idleness is measured
// from the last search, not the last refresh.
const lastSearchedAt = new Map<string, number>();
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Resolved
const indexIdleMillis = Duration.toMillis(WorkspaceSearchIndex.WORKSPACE_INDEX_IDLE_TTL);
const markSearched = (indexKey: string, now: number) => {
// A record older than the timeout counts the same as none, so it can go.
for (const [key, searchedAt] of lastSearchedAt) {
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Resolved
if (now - searchedAt >= indexIdleMillis) lastSearchedAt.delete(key);
}
lastSearchedAt.set(indexKey, now);
};

const refresh: WorkspaceEntries["Service"]["refresh"] = Effect.fn("WorkspaceEntries.refresh")(
function* (cwd) {
const normalizedCwd = yield* normalizeWorkspaceRoot(cwd).pipe(
Expand All @@ -175,6 +189,12 @@ export const make = Effect.gen(function* () {
if (!(yield* RcMap.has(workspaceSearchIndexes.rcMap, indexKey))) {
continue;
}
const searchedAt = lastSearchedAt.get(indexKey) ?? 0;
if ((yield* Clock.currentTimeMillis) - searchedAt >= indexIdleMillis) {
lastSearchedAt.delete(indexKey);
yield* workspaceSearchIndexes.invalidate(indexKey);
continue;
}
const recoverRefreshFailure = Effect.fn("WorkspaceEntries.recoverRefreshFailure")(
function* (
cause:
Expand Down Expand Up @@ -258,16 +278,12 @@ export const make = Effect.gen(function* () {
const normalizedQuery = normalizeSearchQuery(input.query, {
trimLeadingPattern: /^[@./]+/,
});
const indexKey = WorkspaceSearchIndex.workspaceSearchIndexKey(normalizedCwd, "paths");
markSearched(indexKey, yield* Clock.currentTimeMillis);
const result = yield* Effect.gen(function* () {
const searchIndex = yield* WorkspaceSearchIndex.WorkspaceSearchIndex;
return yield* searchIndex.search(normalizedQuery, input.limit, input.kind, input.imageOnly);
}).pipe(
Effect.provide(
workspaceSearchIndexes.get(
WorkspaceSearchIndex.workspaceSearchIndexKey(normalizedCwd, "paths"),
),
),
);
}).pipe(Effect.provide(workspaceSearchIndexes.get(indexKey)));
if (input.kind === "directory" || input.imageOnly) return result;
const worktreeFiles = yield* findNestedWorktreeFiles(
normalizedCwd,
Expand Down Expand Up @@ -356,16 +372,12 @@ export const make = Effect.gen(function* () {
"WorkspaceEntries.searchContents",
)(function* (input) {
const normalizedCwd = yield* normalizeWorkspaceRoot(input.cwd);
const indexKey = WorkspaceSearchIndex.workspaceSearchIndexKey(normalizedCwd, "content");
markSearched(indexKey, yield* Clock.currentTimeMillis);
return yield* Effect.gen(function* () {
const searchIndex = yield* WorkspaceSearchIndex.WorkspaceSearchIndex;
return yield* searchIndex.searchContents(input);
}).pipe(
Effect.provide(
workspaceSearchIndexes.get(
WorkspaceSearchIndex.workspaceSearchIndexKey(normalizedCwd, "content"),
),
),
);
}).pipe(Effect.provide(workspaceSearchIndexes.get(indexKey)));
});

const list: WorkspaceEntries["Service"]["list"] = Effect.fn("WorkspaceEntries.list")(
Expand Down Expand Up @@ -540,16 +552,12 @@ export const make = Effect.gen(function* () {
truncated: false,
};
}
const indexKey = WorkspaceSearchIndex.workspaceSearchIndexKey(normalizedCwd, "paths");
markSearched(indexKey, yield* Clock.currentTimeMillis);
return yield* Effect.gen(function* () {
const searchIndex = yield* WorkspaceSearchIndex.WorkspaceSearchIndex;
return yield* searchIndex.list();
}).pipe(
Effect.provide(
workspaceSearchIndexes.get(
WorkspaceSearchIndex.workspaceSearchIndexKey(normalizedCwd, "paths"),
),
),
);
}).pipe(Effect.provide(workspaceSearchIndexes.get(indexKey)));
},
);

Expand Down
2 changes: 1 addition & 1 deletion apps/server/src/workspace/WorkspaceSearchIndex.ts
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,7 @@ const WORKSPACE_INDEX_MAX_ENTRIES = 25_000;
const WORKSPACE_INDEX_PAGE_SIZE = WORKSPACE_INDEX_MAX_ENTRIES + 2;
const WORKSPACE_INDEX_SCAN_TIMEOUT = "15 seconds";
const WORKSPACE_INDEX_SCAN_TIMEOUT_MS = 15_000;
const WORKSPACE_INDEX_IDLE_TTL = "15 minutes";
export const WORKSPACE_INDEX_IDLE_TTL = "15 minutes";
const CONTENT_SEARCH_TIME_BUDGET_MS = 250;
const CONTENT_SEARCH_MAX_MATCHES_PER_FILE = 100;
const GIT_GREP_MAX_OUTPUT_BYTES = 4 * 1024 * 1024;
Expand Down
Loading