From d9e80ce0890384af95aaa3988c599219a542af90 Mon Sep 17 00:00:00 2001 From: Badcuban <108198679+badcuban@users.noreply.github.com> Date: Sat, 21 Nov 2026 01:02:40 -0500 Subject: [PATCH] perf(server): a folder's repo root is remembered, not asked of git on every lookup RepositoryIdentityResolver ran `git rev-parse --show-toplevel` on every resolve, because git's answer was the identity cache's key. The pull request sweep alone made 32 of those per pass over 11 folders, every 20 to 120 seconds, about a third of the child processes the server starts. The resolver now remembers each folder's root and confirms it on disk: the folder still resolves to the same real path, no folder between it and the root holds a `.git` or a `HEAD`, and the root's `.git` exists. When a check fails it asks git again at once, and it re-asks every 10 minutes regardless. An answer the disk can't confirm is never reused. Concurrent resolves of one folder share one git run. A sweep now makes 11 lookups on its first pass and none after. --- .../Layers/RepositoryIdentityResolver.test.ts | 205 ++++++++++++++++++ .../Layers/RepositoryIdentityResolver.ts | 185 ++++++++++++++-- 2 files changed, 374 insertions(+), 16 deletions(-) diff --git a/apps/server/src/project/Layers/RepositoryIdentityResolver.test.ts b/apps/server/src/project/Layers/RepositoryIdentityResolver.test.ts index 78b67e4d3..4d31c067e 100644 --- a/apps/server/src/project/Layers/RepositoryIdentityResolver.test.ts +++ b/apps/server/src/project/Layers/RepositoryIdentityResolver.test.ts @@ -57,6 +57,40 @@ const makeRepositoryIdentityResolverTestLayer = (options: { }), ).pipe(Layer.provide(ProcessRunner.layer)); +/** + * A resolver whose git runs are recorded in `runs`, oldest first, so a test + * can count how many processes a sequence of resolves started. + */ +const makeCountingResolverLayer = ( + runs: Array>, + options: { + readonly positiveCacheTtl?: Duration.Input; + readonly negativeCacheTtl?: Duration.Input; + readonly rootRecheckInterval?: Duration.Input; + } = {}, +) => + Layer.effect( + RepositoryIdentityResolver, + makeRepositoryIdentityResolver({ cacheCapacity: 16, ...options }), + ).pipe( + Layer.provide( + Layer.effect( + ProcessRunner.ProcessRunner, + Effect.map(ProcessRunner.make(), (real) => + ProcessRunner.ProcessRunner.of({ + run: (input) => { + runs.push(input.args); + return real.run(input); + }, + }), + ), + ), + ), + ); + +const countGit = (runs: ReadonlyArray>, subcommand: "rev-parse" | "remote") => + runs.filter((args) => args.includes(subcommand)).length; + it.layer(NodeServices.layer)("RepositoryIdentityResolverLive", (it) => { it.effect("normalizes equivalent GitHub remotes into a stable repository identity", () => Effect.gen(function* () { @@ -247,4 +281,175 @@ it.layer(NodeServices.layer)("RepositoryIdentityResolverLive", (it) => { ), ), ); + + it.effect("asks git for a folder's root once, however often and concurrently it resolves", () => { + const runs: Array> = []; + return Effect.gen(function* () { + const fileSystem = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const repoRoot = yield* fileSystem.makeTempDirectoryScoped({ + prefix: "t3-repository-identity-repeat-test-", + }); + const nestedWorkspace = path.join(repoRoot, "packages", "web"); + yield* fileSystem.makeDirectory(nestedWorkspace, { recursive: true }); + yield* git(repoRoot, ["init"]); + yield* git(repoRoot, ["remote", "add", "origin", "git@github.com:T3Tools/t3code.git"]); + + const resolver = yield* RepositoryIdentityResolver; + const identities = yield* Effect.all( + [repoRoot, repoRoot, nestedWorkspace, nestedWorkspace, repoRoot, nestedWorkspace].map( + (cwd) => resolver.resolve(cwd), + ), + { concurrency: "unbounded" }, + ); + for (const _sweep of [1, 2, 3]) { + identities.push( + yield* resolver.resolve(repoRoot), + yield* resolver.resolve(nestedWorkspace), + ); + } + + expect(identities.map((identity) => identity?.canonicalKey)).toEqual( + identities.map(() => "github.com/t3tools/t3code"), + ); + expect(countGit(runs, "rev-parse")).toBe(2); + expect(countGit(runs, "remote")).toBe(1); + + // A remembered root is still re-asked once the recheck interval passes. + yield* TestClock.adjust(Duration.minutes(5)); + yield* resolver.resolve(repoRoot); + expect(countGit(runs, "rev-parse")).toBe(3); + }).pipe( + Effect.provide( + Layer.merge( + TestClock.layer(), + makeCountingResolverLayer(runs, { + positiveCacheTtl: Duration.minutes(10), + rootRecheckInterval: Duration.minutes(5), + }), + ), + ), + ); + }); + + it.effect("finds a repository created inside a remembered root on the next resolve", () => { + const runs: Array> = []; + return Effect.gen(function* () { + const fileSystem = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const repoRoot = yield* fileSystem.makeTempDirectoryScoped({ + prefix: "t3-repository-identity-nested-repo-test-", + }); + const nestedWorkspace = path.join(repoRoot, "vendor", "lib"); + yield* fileSystem.makeDirectory(nestedWorkspace, { recursive: true }); + yield* git(repoRoot, ["init"]); + yield* git(repoRoot, ["remote", "add", "origin", "git@github.com:T3Tools/t3code.git"]); + + const resolver = yield* RepositoryIdentityResolver; + expect((yield* resolver.resolve(nestedWorkspace))?.canonicalKey).toBe( + "github.com/t3tools/t3code", + ); + + yield* git(nestedWorkspace, ["init"]); + yield* git(nestedWorkspace, ["remote", "add", "origin", "git@github.com:T3Tools/lib.git"]); + + const nestedIdentity = yield* resolver.resolve(nestedWorkspace); + expect(nestedIdentity?.canonicalKey).toBe("github.com/t3tools/lib"); + expectSameDirectory(nestedIdentity?.rootPath ?? "", nestedWorkspace); + expect(countGit(runs, "rev-parse")).toBe(2); + }).pipe(Effect.provide(Layer.merge(TestClock.layer(), makeCountingResolverLayer(runs)))); + }); + + it.effect("finds a bare repository created inside a remembered root on the next resolve", () => { + const runs: Array> = []; + return Effect.gen(function* () { + const fileSystem = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const repoRoot = yield* fileSystem.makeTempDirectoryScoped({ + prefix: "t3-repository-identity-nested-bare-test-", + }); + const nestedWorkspace = path.join(repoRoot, "vendor"); + yield* fileSystem.makeDirectory(nestedWorkspace); + yield* git(repoRoot, ["init"]); + yield* git(repoRoot, ["remote", "add", "origin", "git@github.com:T3Tools/t3code.git"]); + + const resolver = yield* RepositoryIdentityResolver; + expect((yield* resolver.resolve(nestedWorkspace))?.canonicalKey).toBe( + "github.com/t3tools/t3code", + ); + + yield* git(nestedWorkspace, ["init", "--bare"]); + yield* git(nestedWorkspace, ["remote", "add", "origin", "git@github.com:T3Tools/lib.git"]); + + expect((yield* resolver.resolve(nestedWorkspace))?.canonicalKey).toBe( + "github.com/t3tools/lib", + ); + }).pipe(Effect.provide(Layer.merge(TestClock.layer(), makeCountingResolverLayer(runs)))); + }); + + it.effect("asks git again once a folder that had no repository gets one", () => { + const runs: Array> = []; + return Effect.gen(function* () { + const fileSystem = yield* FileSystem.FileSystem; + const cwd = yield* fileSystem.makeTempDirectoryScoped({ + prefix: "t3-repository-identity-git-init-test-", + }); + + const resolver = yield* RepositoryIdentityResolver; + expect(yield* resolver.resolve(cwd)).toBeNull(); + expect(yield* resolver.resolve(cwd)).toBeNull(); + expect(countGit(runs, "rev-parse")).toBe(1); + + yield* git(cwd, ["init"]); + yield* git(cwd, ["remote", "add", "origin", "git@github.com:T3Tools/t3code.git"]); + yield* resolver.resolve(cwd); + expect(countGit(runs, "rev-parse")).toBe(2); + + // Where git spells the new root the same as the folder, the cached + // "no identity" answer holds until the negative TTL, as before. + yield* TestClock.adjust(Duration.millis(120)); + expect((yield* resolver.resolve(cwd))?.canonicalKey).toBe("github.com/t3tools/t3code"); + expect(countGit(runs, "rev-parse")).toBe(2); + }).pipe( + Effect.provide( + Layer.merge( + TestClock.layer(), + makeCountingResolverLayer(runs, { negativeCacheTtl: Duration.millis(100) }), + ), + ), + ); + }); + + it.effect("picks up a folder re-created as another repository once the identity TTL ends", () => { + const runs: Array> = []; + return Effect.gen(function* () { + const fileSystem = yield* FileSystem.FileSystem; + const cwd = yield* fileSystem.makeTempDirectoryScoped({ + prefix: "t3-repository-identity-recreated-test-", + }); + yield* git(cwd, ["init"]); + yield* git(cwd, ["remote", "add", "origin", "git@github.com:T3Tools/t3code.git"]); + + const resolver = yield* RepositoryIdentityResolver; + expect((yield* resolver.resolve(cwd))?.canonicalKey).toBe("github.com/t3tools/t3code"); + + yield* fileSystem.remove(cwd, { recursive: true }); + yield* fileSystem.makeDirectory(cwd); + yield* git(cwd, ["init"]); + yield* git(cwd, ["remote", "add", "origin", "git@github.com:T3Tools/other.git"]); + + expect((yield* resolver.resolve(cwd))?.canonicalKey).toBe("github.com/t3tools/t3code"); + yield* TestClock.adjust(Duration.millis(120)); + expect((yield* resolver.resolve(cwd))?.canonicalKey).toBe("github.com/t3tools/other"); + // The root never changed, so git was asked for it only once. + expect(countGit(runs, "rev-parse")).toBe(1); + }).pipe( + Effect.provide( + Layer.merge( + TestClock.layer(), + makeCountingResolverLayer(runs, { positiveCacheTtl: Duration.millis(100) }), + ), + ), + ); + }); }); diff --git a/apps/server/src/project/Layers/RepositoryIdentityResolver.ts b/apps/server/src/project/Layers/RepositoryIdentityResolver.ts index 1b13f679c..ab4edeed4 100644 --- a/apps/server/src/project/Layers/RepositoryIdentityResolver.ts +++ b/apps/server/src/project/Layers/RepositoryIdentityResolver.ts @@ -3,11 +3,14 @@ import * as Cache from "effect/Cache"; import * as Duration from "effect/Duration"; import * as Effect from "effect/Effect"; import * as Exit from "effect/Exit"; +import * as FileSystem from "effect/FileSystem"; import * as Layer from "effect/Layer"; +import * as Path from "effect/Path"; import { detectSourceControlProviderFromGitRemoteUrl, normalizeGitRemoteUrl, } from "@threadlines/shared/git"; +import { areFilesystemPathsEqual } from "@threadlines/shared/path"; import * as ProcessRunner from "../../processRunner.ts"; import { @@ -76,19 +79,122 @@ function buildRepositoryIdentity(input: { const DEFAULT_REPOSITORY_IDENTITY_CACHE_CAPACITY = 512; const DEFAULT_POSITIVE_CACHE_TTL = Duration.minutes(1); const DEFAULT_NEGATIVE_CACHE_TTL = Duration.minutes(1); +/** + * How long a remembered repository root is reused before git is asked again, + * even while its `.git` check still passes. A safety net for the changes the + * check can't see, such as a folder's owner changing or `core.worktree` + * being set. + */ +const DEFAULT_ROOT_RECHECK_INTERVAL = Duration.minutes(10); interface RepositoryIdentityResolverOptions { readonly cacheCapacity?: number; readonly positiveCacheTtl?: Duration.Input; readonly negativeCacheTtl?: Duration.Input; + readonly rootRecheckInterval?: Duration.Input; } -const resolveRepositoryIdentityCacheKey = Effect.fn("resolveRepositoryIdentityCacheKey")(function* ( +/** + * The entries on disk that decided git's answer for one folder: a `.git` in + * the top level, and no repository in any folder below it (no `.git`, and no + * `HEAD`, which marks a bare repository). While each is still as recorded, git + * would give the same answer, so the answer is reused without starting git. + */ +interface RepositoryRootCheck { + /** The folder with symlinks resolved, as it was when git was asked. */ + readonly realCwd: string; + /** The folder and its parents, up to the top level, that held no repository. */ + readonly withoutGit: ReadonlyArray; + /** The top level, whose `.git` must still exist; null outside a repository. */ + readonly withGit: string | null; +} + +/** + * Which repository root a folder belongs to. `root` keys the identity cache: + * git's top level, or the folder itself outside a repository. `check` is null + * when the answer must not be reused: git didn't finish, or the `.git` entries + * on disk don't explain its answer. + */ +interface RepositoryRootLookup { + readonly root: string; + readonly check: RepositoryRootCheck | null; +} + +const makeRepositoryRootChecks = Effect.fn("makeRepositoryRootChecks")(function* () { + const fileSystem = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + + /** Null when the disk can't be read, which counts as a change. */ + const hasEntry = (directory: string, name: ".git" | "HEAD") => + fileSystem.exists(path.join(directory, name)).pipe(Effect.orElseSucceed(() => null)); + + /** Whether the entries recorded in `check` are all as they were. */ + const stillHolds = Effect.fn("RepositoryRootCheck.stillHolds")(function* ( + cwd: string, + check: RepositoryRootCheck, + ) { + const realCwd = yield* fileSystem.realPath(cwd).pipe(Effect.orElseSucceed(() => null)); + if (realCwd !== check.realCwd) { + return false; + } + for (const directory of check.withoutGit) { + if ( + (yield* hasEntry(directory, ".git")) !== false || + (yield* hasEntry(directory, "HEAD")) !== false + ) { + return false; + } + } + return check.withGit === null || (yield* hasEntry(check.withGit, ".git")) === true; + }); + + /** + * Records the folders between `cwd` and git's top level, or every parent of + * `cwd` when git found no repository. Null when the top level is not one of + * those folders (a symlink git resolved differently) or when the disk + * disagrees with git right now (a `.git` git skipped, as in a clone still + * being written), since such an answer can't be confirmed later. + */ + const record = Effect.fn("RepositoryRootCheck.record")(function* ( + cwd: string, + topLevel: string | null, + ) { + const realCwd = yield* fileSystem.realPath(cwd).pipe(Effect.orElseSucceed(() => null)); + if (realCwd === null) { + return null; + } + // Git prints `/` on Windows, `C:/repo` and `//server/share/repo`. + const topLevelHere = topLevel?.replaceAll("/", path.sep) ?? null; + const withoutGit: Array = []; + let withGit: string | null = null; + let directory = realCwd; + while (true) { + if (topLevelHere !== null && areFilesystemPathsEqual(directory, topLevelHere)) { + withGit = directory; + break; + } + withoutGit.push(directory); + const parent = path.dirname(directory); + if (parent === directory) { + if (topLevelHere !== null) { + return null; + } + break; + } + directory = parent; + } + const check: RepositoryRootCheck = { realCwd, withoutGit, withGit }; + return (yield* stillHolds(cwd, check)) ? check : null; + }); + + return { record, stillHolds }; +}); + +const resolveRepositoryRoot = Effect.fn("resolveRepositoryIdentityCacheKey")(function* ( cwd: string, -) { + checks: Effect.Success>, +): Effect.fn.Return { const processRunner = yield* ProcessRunner.ProcessRunner; - let cacheKey = cwd; - const topLevelResult = yield* processRunner .run({ command: "git", @@ -96,16 +202,17 @@ const resolveRepositoryIdentityCacheKey = Effect.fn("resolveRepositoryIdentityCa timeoutBehavior: "timedOutResult", }) .pipe(Effect.option); - if (topLevelResult._tag === "None" || topLevelResult.value.code !== 0) { - return cacheKey; + if (topLevelResult._tag === "None" || topLevelResult.value.timedOut) { + return { root: cwd, check: null }; } - - const candidate = topLevelResult.value.stdout.trim(); - if (candidate.length > 0) { - cacheKey = candidate; + if (topLevelResult.value.code !== 0) { + return { root: cwd, check: yield* checks.record(cwd, null) }; } - - return cacheKey; + const topLevel = topLevelResult.value.stdout.trim(); + if (topLevel.length === 0) { + return { root: cwd, check: null }; + } + return { root: topLevel, check: yield* checks.record(cwd, topLevel) }; }); const resolveRepositoryIdentityFromCacheKey = Effect.fn("resolveRepositoryIdentityFromCacheKey")( @@ -129,9 +236,44 @@ const resolveRepositoryIdentityFromCacheKey = Effect.fn("resolveRepositoryIdenti }, ); +/** + * Resolves a folder to the repository identity of the git repository it sits + * in, in two cached steps. + * + * 1. Folder to repository root. Git is asked once; after that the answer is + * confirmed by checking the same `.git` entries on disk, so a folder that + * is re-resolved every sweep starts no process. A new repository between + * the folder and its root (a nested repo, bare or not, or `git init`), or + * the root's `.git` disappearing, sends the next resolve back to git at + * once. Remembered + * roots are re-asked at least every `rootRecheckInterval`. + * 2. Root to identity, from `git remote -v`, kept for the positive or negative + * TTL. A changed remote URL shows up once that TTL runs out. + * + * Concurrent resolves of one folder share one git run in each step. + */ export const makeRepositoryIdentityResolver = Effect.fn("makeRepositoryIdentityResolver")( function* (options: RepositoryIdentityResolverOptions = {}) { const processRunner = yield* ProcessRunner.ProcessRunner; + const checks = yield* makeRepositoryRootChecks(); + const capacity = options.cacheCapacity ?? DEFAULT_REPOSITORY_IDENTITY_CACHE_CAPACITY; + + const repositoryRootCache = yield* Cache.makeWith( + (cwd) => + resolveRepositoryRoot(cwd, checks).pipe( + Effect.provideService(ProcessRunner.ProcessRunner, processRunner), + ), + { + capacity, + timeToLive: Exit.match({ + onSuccess: (lookup) => + lookup.check === null + ? Duration.zero + : (options.rootRecheckInterval ?? DEFAULT_ROOT_RECHECK_INTERVAL), + onFailure: () => Duration.zero, + }), + }, + ); const repositoryIdentityCache = yield* Cache.makeWith( (cacheKey) => @@ -139,7 +281,7 @@ export const makeRepositoryIdentityResolver = Effect.fn("makeRepositoryIdentityR Effect.provideService(ProcessRunner.ProcessRunner, processRunner), ), { - capacity: options.cacheCapacity ?? DEFAULT_REPOSITORY_IDENTITY_CACHE_CAPACITY, + capacity, timeToLive: Exit.match({ onSuccess: (value) => value === null @@ -150,12 +292,23 @@ export const makeRepositoryIdentityResolver = Effect.fn("makeRepositoryIdentityR }, ); + const resolveRoot = Effect.fn("RepositoryIdentityResolver.resolveRoot")(function* ( + cwd: string, + ) { + const remembered = yield* Cache.get(repositoryRootCache, cwd); + if (remembered.check === null || (yield* checks.stillHolds(cwd, remembered.check))) { + return remembered.root; + } + // Drops only the entry just checked, so callers that found the same + // stale entry together share the one git run that replaces it. + yield* Cache.invalidateWhen(repositoryRootCache, cwd, (current) => current === remembered); + return (yield* Cache.get(repositoryRootCache, cwd)).root; + }); + const resolve: RepositoryIdentityResolverShape["resolve"] = Effect.fn( "RepositoryIdentityResolver.resolve", )(function* (cwd) { - const cacheKey = yield* resolveRepositoryIdentityCacheKey(cwd).pipe( - Effect.provideService(ProcessRunner.ProcessRunner, processRunner), - ); + const cacheKey = yield* resolveRoot(cwd); return yield* Cache.get(repositoryIdentityCache, cacheKey); });