From 735f10bb0e0dbedf77b8f78d28a95bda694de655 Mon Sep 17 00:00:00 2001 From: max Date: Sat, 17 Oct 2026 03:37:41 +0000 Subject: [PATCH 1/3] Serve the workspace with gitfs behind BLACKSMITH_USE_GITFS When the job's installation has use_gitfs_checkout on, mount the requested revision from the git mirror sticky disk with gitfs instead of fetching and checking it out, create .git through the mount so git works from the first step, and take gitfs down in the post step before the mirror is unmounted. Anything outside the plain Linux cohort, or any gitfs failure, falls through to the regular checkout. Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- BLACKSMITH.md | 33 ++ __test__/gitfs.test.ts | 290 ++++++++++++++ dist/index.js | 748 +++++++++++++++++++++++++++++-------- src/blacksmith-cache.ts | 2 +- src/git-source-provider.ts | 490 +++++++++++++++--------- src/gitfs.ts | 357 ++++++++++++++++++ src/main.ts | 11 + src/state-helper.ts | 10 + 8 files changed, 1611 insertions(+), 330 deletions(-) create mode 100644 __test__/gitfs.test.ts create mode 100644 src/gitfs.ts diff --git a/BLACKSMITH.md b/BLACKSMITH.md index b271978..4c5af00 100644 --- a/BLACKSMITH.md +++ b/BLACKSMITH.md @@ -307,6 +307,39 @@ The mirror stays in sync with GitHub through **incremental fetches**: - Redundant pack files (consolidated into single optimized pack) - All unreachable objects (removed immediately via `git gc --prune=now`) +## Serving the Workspace with gitfs + +When the job's installation has `use_gitfs_checkout` enabled, the agent +exports `BLACKSMITH_USE_GITFS=true` into the runner environment and the +action serves the workspace from the mirror with +[gitfs](https://github.com/useblacksmith/gitfs) instead of fetching and +checking it out: + +1. The requested ref/commit is resolved in the mirror. If the mirror does not + have it yet (a push it has not synced, a pull-request merge ref), only that + ref is fetched *into the mirror*, where the regular path would have fetched + it into the workspace. +2. `gitfs mount --ref $GITHUB_WORKSPACE --daemon` mounts the + tree in milliseconds. gitfs writes the real checkout out in the background + and atomically swaps it in for the mount once it is complete (seconds, even + for 100k-file trees); until then reads are served lazily from the mirror's + packs and writes go straight into the tree being written. +3. `.git` is created through the mount exactly as in the regular path + (`git init`, alternates to the mirror, `origin`, auth, the mirror's + branch/tag refs, the local branch or detached HEAD the checkout would have + produced) plus `git read-tree ` for the index. Git is usable from the + first step; the first `git status` verifies file contents instead of + trusting stat data. +4. The post step takes gitfs down before the mirror sticky disk is unmounted. + If the job ends before the swap, the partial tree is discarded. + +gitfs is only used for the plain case — Linux runner VM (not inside a +container), the mirror mounted, and no submodules, LFS, sparse checkout, +`filter` or `dissociate` — and only when the `gitfs` binary is on the PATH. +Everything else, and any gitfs failure, falls through to the regular fetch + +checkout. Because objects come from the mirror through alternates, the +workspace has the full history regardless of `fetch-depth`. + ## The `dissociate` Option When to use `dissociate: true`: diff --git a/__test__/gitfs.test.ts b/__test__/gitfs.test.ts new file mode 100644 index 0000000..dcc78ba --- /dev/null +++ b/__test__/gitfs.test.ts @@ -0,0 +1,290 @@ +// gitfs.ts pulls in blacksmith-cache for the auth header; stub its gRPC +// dependencies the same way blacksmith-cache.test.ts does. +jest.mock('@connectrpc/connect', () => ({ + createClient: jest.fn(), + ConnectError: class ConnectError extends Error {}, + Code: {Aborted: 'ABORTED'} +})) +jest.mock('@connectrpc/connect-node', () => ({ + createGrpcTransport: jest.fn() +})) +jest.mock( + '@buf/blacksmith_vm-agent.connectrpc_es/stickydisk/v1/stickydisk_connect', + () => ({StickyDiskService: {}}) +) + +import * as fs from 'fs' +import * as os from 'os' +import * as path from 'path' +import {execFileSync} from 'child_process' +import * as gitfs from '../src/gitfs' + +const settings = ( + overrides: Partial = {} +): gitfs.EligibilitySettings => ({ + ref: 'refs/heads/main', + commit: '', + submodules: false, + lfs: false, + sparseCheckout: [], + filter: undefined, + dissociate: false, + repositoryPath: path.join(os.tmpdir(), 'gitfs-test-does-not-exist'), + ...overrides +}) + +const ctx: gitfs.EligibilityContext = { + mirrorPath: '/mnt/git-mirror/owner-repo.git', + platform: 'linux', + inContainer: false +} + +describe('isGitfsRequested', () => { + it('is driven by BLACKSMITH_USE_GITFS=true only', () => { + expect(gitfs.isGitfsRequested({})).toBe(false) + expect(gitfs.isGitfsRequested({BLACKSMITH_USE_GITFS: 'false'})).toBe(false) + expect(gitfs.isGitfsRequested({BLACKSMITH_USE_GITFS: 'TRUE'})).toBe(false) + expect(gitfs.isGitfsRequested({BLACKSMITH_USE_GITFS: 'true'})).toBe(true) + }) +}) + +describe('gitfsIneligibilityReason', () => { + it('accepts the plain linux + mirror case', () => { + expect(gitfs.gitfsIneligibilityReason(settings(), ctx)).toBeNull() + expect( + gitfs.gitfsIneligibilityReason( + settings({ref: '', commit: 'a'.repeat(40)}), + ctx + ) + ).toBeNull() + }) + + it('needs a mounted mirror on a linux runner outside a container', () => { + expect( + gitfs.gitfsIneligibilityReason(settings(), {...ctx, mirrorPath: null}) + ).toMatch(/mirror/) + expect( + gitfs.gitfsIneligibilityReason(settings(), {...ctx, platform: 'darwin'}) + ).toMatch(/linux/) + expect( + gitfs.gitfsIneligibilityReason(settings(), {...ctx, inContainer: true}) + ).toMatch(/container/) + }) + + it('leaves features the fetch path implements to the fetch path', () => { + expect( + gitfs.gitfsIneligibilityReason(settings({submodules: true}), ctx) + ).toMatch(/submodules/) + expect(gitfs.gitfsIneligibilityReason(settings({lfs: true}), ctx)).toMatch( + /lfs/ + ) + expect( + gitfs.gitfsIneligibilityReason(settings({sparseCheckout: ['src']}), ctx) + ).toMatch(/sparse/) + expect( + gitfs.gitfsIneligibilityReason(settings({filter: 'blob:none'}), ctx) + ).toMatch(/filter/) + expect( + gitfs.gitfsIneligibilityReason(settings({dissociate: true}), ctx) + ).toMatch(/dissociate/) + expect( + gitfs.gitfsIneligibilityReason(settings({ref: '', commit: ''}), ctx) + ).toMatch(/ref or commit/) + }) + + it('does not mount over an existing repository', () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'gitfs-elig-')) + try { + fs.mkdirSync(path.join(dir, '.git')) + expect( + gitfs.gitfsIneligibilityReason(settings({repositoryPath: dir}), ctx) + ).toMatch(/already contains/) + } finally { + fs.rmSync(dir, {recursive: true, force: true}) + } + }) +}) + +describe('mirrorRevisionCandidates', () => { + const sha = 'b'.repeat(40) + + it('prefers the commit and keeps the ref it belongs to', () => { + expect(gitfs.mirrorRevisionCandidates('refs/heads/main', sha)).toEqual([ + {rev: `${sha}^{commit}`, mirrorRef: 'refs/heads/main', kind: 'branch'} + ]) + expect(gitfs.mirrorRevisionCandidates('refs/pull/42/merge', sha)).toEqual([ + {rev: `${sha}^{commit}`, mirrorRef: 'refs/pull/42/merge', kind: 'pull'} + ]) + expect(gitfs.mirrorRevisionCandidates('', sha)).toEqual([ + {rev: `${sha}^{commit}`, mirrorRef: '', kind: 'commit'} + ]) + // An unqualified ref alongside a commit does not name a mirror ref. + expect(gitfs.mirrorRevisionCandidates('main', sha)).toEqual([ + {rev: `${sha}^{commit}`, mirrorRef: '', kind: 'other'} + ]) + }) + + it('peels qualified refs to commits', () => { + expect(gitfs.mirrorRevisionCandidates('refs/tags/v1', '')).toEqual([ + {rev: 'refs/tags/v1^{commit}', mirrorRef: 'refs/tags/v1', kind: 'tag'} + ]) + }) + + it('tries an unqualified ref as a branch, then a tag', () => { + expect( + gitfs.mirrorRevisionCandidates('v1', '').map(c => c.mirrorRef) + ).toEqual(['refs/heads/v1', 'refs/tags/v1']) + expect(gitfs.mirrorRevisionCandidates('', '')).toEqual([]) + }) +}) + +describe('mirrorFetchRefspecs', () => { + const sha = 'c'.repeat(40) + it('fetches qualified refs under their own name and bare commits by sha', () => { + expect(gitfs.mirrorFetchRefspecs('refs/pull/7/merge', sha)).toEqual([ + `+${sha}:refs/pull/7/merge` + ]) + expect(gitfs.mirrorFetchRefspecs('refs/heads/dev', '')).toEqual([ + '+refs/heads/dev:refs/heads/dev' + ]) + expect(gitfs.mirrorFetchRefspecs('', sha)).toEqual([sha]) + expect(gitfs.mirrorFetchRefspecs('dev', sha)).toEqual([sha]) + expect(gitfs.mirrorFetchRefspecs('dev', '')).toEqual([ + '+refs/heads/dev*:refs/heads/dev*', + '+refs/tags/dev*:refs/tags/dev*' + ]) + }) +}) + +// End-to-end against a real mirror and the gitfs binary (skipped when gitfs +// is not on PATH): mount, give the workspace its .git through the mount, and +// check git is usable before and after the swap to the native tree. +const hasGitfs = ((): boolean => { + try { + execFileSync('gitfs', ['--help'], {stdio: 'ignore'}) + return true + } catch { + return false + } +})() + +const describeE2E = + hasGitfs && process.platform === 'linux' ? describe : describe.skip + +describeE2E('gitfs mount of a --mirror clone', () => { + jest.setTimeout(120_000) + let root: string + let mirror: string + let workspace: string + const git = (cwd: string, args: string[]): string => + execFileSync('git', ['-C', cwd, ...args], { + encoding: 'utf8', + env: { + ...process.env, + GIT_AUTHOR_NAME: 't', + GIT_AUTHOR_EMAIL: 't@t', + GIT_COMMITTER_NAME: 't', + GIT_COMMITTER_EMAIL: 't@t' + } + }).trim() + + beforeAll(() => { + root = fs.mkdtempSync(path.join(os.tmpdir(), 'gitfs-e2e-')) + const upstream = path.join(root, 'upstream') + fs.mkdirSync(upstream) + git(upstream, ['init', '-q', '-b', 'main']) + fs.mkdirSync(path.join(upstream, 'src')) + fs.writeFileSync(path.join(upstream, 'src', 'a.txt'), 'a\n') + fs.writeFileSync(path.join(upstream, 'README'), 'hi\n') + git(upstream, ['add', '.']) + git(upstream, ['commit', '-q', '-m', 'one']) + git(upstream, ['tag', 'v1']) + fs.writeFileSync(path.join(upstream, 'src', 'b.txt'), 'b\n') + git(upstream, ['add', '.']) + git(upstream, ['commit', '-q', '-m', 'two']) + mirror = path.join(root, 'owner-repo.git') + execFileSync('git', ['clone', '-q', '--mirror', upstream, mirror]) + workspace = path.join(root, 'work', 'repo') + fs.mkdirSync(workspace, {recursive: true}) + }) + + afterAll(async () => { + if (workspace) { + await gitfs.teardown(workspace, mirror) + } + fs.rmSync(root, {recursive: true, force: true}) + }) + + it('serves the branch tip with a working .git from the first command', async () => { + const binary = await gitfs.findBinary() + expect(binary).not.toBeNull() + const revision = await gitfs.resolveInMirror( + mirror, + gitfs.mirrorRevisionCandidates('refs/heads/main', '') + ) + expect(revision).toEqual({ + sha: git(mirror, ['rev-parse', 'refs/heads/main']), + mirrorRef: 'refs/heads/main', + kind: 'branch' + }) + + await gitfs.mount(binary!, mirror, revision!.sha, workspace) + expect(fs.readFileSync(path.join(workspace, 'src', 'b.txt'), 'utf8')).toBe( + 'b\n' + ) + + git(workspace, ['init', '-q']) + fs.mkdirSync(path.join(workspace, '.git', 'objects', 'info'), { + recursive: true + }) + fs.writeFileSync( + path.join(workspace, '.git', 'objects', 'info', 'alternates'), + `${mirror}/objects\n` + ) + git(workspace, ['remote', 'add', 'origin', mirror]) + execFileSync('git', [ + '-C', + workspace, + 'fetch', + '-q', + '--no-tags', + mirror, + '+refs/heads/*:refs/remotes/origin/*', + '+refs/tags/*:refs/tags/*' + ]) + await gitfs.pointWorkspaceAt(workspace, 'refs/heads/main', revision!) + + expect(git(workspace, ['rev-parse', 'HEAD'])).toBe(revision!.sha) + expect(git(workspace, ['symbolic-ref', 'HEAD'])).toBe('refs/heads/main') + expect(git(workspace, ['rev-parse', 'refs/remotes/origin/main'])).toBe( + revision!.sha + ) + expect(git(workspace, ['status', '--porcelain'])).toBe('') + expect(git(workspace, ['log', '--oneline']).split('\n')).toHaveLength(2) + + // Wait for the swap: the workspace path becomes a plain directory. + const deadline = Date.now() + 60_000 + while (fs.lstatSync(workspace).isSymbolicLink() && Date.now() < deadline) { + await new Promise(r => setTimeout(r, 50)) + } + expect(fs.lstatSync(workspace).isDirectory()).toBe(true) + expect(git(workspace, ['status', '--porcelain'])).toBe('') + expect(git(workspace, ['rev-parse', 'HEAD'])).toBe(revision!.sha) + expect(fs.readFileSync(path.join(workspace, 'src', 'a.txt'), 'utf8')).toBe( + 'a\n' + ) + + // The mirror itself is untouched. + expect( + fs.existsSync(path.join(mirror, 'objects', 'info', 'alternates')) + ).toBe(false) + }) + + it('refuses a non-empty mountpoint', async () => { + const dir = fs.mkdtempSync(path.join(root, 'nonempty-')) + fs.writeFileSync(path.join(dir, 'x'), '') + await expect(gitfs.mount('gitfs', mirror, 'HEAD', dir)).rejects.toThrow( + /not empty/ + ) + }) +}) diff --git a/dist/index.js b/dist/index.js index d1d1519..87297cf 100644 --- a/dist/index.js +++ b/dist/index.js @@ -49,6 +49,7 @@ exports.shouldUseBlacksmithCache = shouldUseBlacksmithCache; exports.getMirrorPath = getMirrorPath; exports.stickyDiskKeyFor = stickyDiskKeyFor; exports.setupCache = setupCache; +exports.getAuthConfigArgs = getAuthConfigArgs; exports.ensureMirror = ensureMirror; exports.diffMirrorRefs = diffMirrorRefs; exports.purgePullRefs = purgePullRefs; @@ -3603,6 +3604,7 @@ const stateHelper = __importStar(__nccwpck_require__(4866)); const urlHelper = __importStar(__nccwpck_require__(9437)); const blacksmithCache = __importStar(__nccwpck_require__(9242)); const mirrorTelemetry = __importStar(__nccwpck_require__(7185)); +const gitfs = __importStar(__nccwpck_require__(1574)); const git_command_manager_1 = __nccwpck_require__(738); function getSource(settings) { return __awaiter(this, void 0, void 0, function* () { @@ -3810,6 +3812,23 @@ function getSourceInner(settings, report, onHydrationReport) { cacheInfo = null; } } + // Serve the workspace with gitfs when the job asks for it: mount the + // requested revision from the mirror now (milliseconds), and let the + // tree be written out behind the job. The .git set up below is created + // through the mount, so git works from the first step. Anything gitfs + // cannot serve falls through to the regular fetch + checkout. + let gitfsRevision = null; + if (gitfs.isGitfsRequested()) { + core.startGroup('Mounting the repository with gitfs'); + try { + gitfsRevision = yield mountWithGitfs(settings, cacheInfo, repositoryUrl); + } + catch (error) { + core.warning(`[gitfs] mount failed, using the regular checkout: ${error}`); + gitfsRevision = null; + } + core.endGroup(); + } // Initialize the repository if (!fsHelper.directoryExistsSync(path.join(settings.repositoryPath, '.git'))) { core.startGroup('Initializing the repository'); @@ -3854,157 +3873,21 @@ function getSourceInner(settings, report, onHydrationReport) { if (settings.lfs) { yield git.lfsInstall(); } - // Fetch - core.startGroup('Fetching the repository'); - const objectsDir = path.join(settings.repositoryPath, '.git', 'objects'); - const objectsBytesBefore = cacheInfo - ? yield mirrorTelemetry.dirSizeBytesOrNull(objectsDir) - : null; - const fetchStart = Date.now(); - const fetchOptions = {}; - if (settings.filter) { - fetchOptions.filter = settings.filter; - } - else if (settings.sparseCheckout) { - fetchOptions.filter = 'blob:none'; - } - if (settings.fetchDepth <= 0) { - // When the Blacksmith mirror is available and synced with the remote, - // copy branch/tag refs from it locally (objects are already shared via - // alternates) instead of doing a full +refs/heads/* fetch over the - // network, and only ask the network for the specific ref/commit this - // run needs. - let fetchedFromMirror = false; - if (cacheInfo && - mirrorFresh && - !fetchOptions.filter && - !fsHelper.fileExistsSync(path.join(settings.repositoryPath, '.git', 'shallow'))) { - fetchedFromMirror = yield blacksmithCache.fetchRefsFromMirror(settings.repositoryPath, cacheInfo.mirrorPath, git.getEnvironment()); - } - if (fetchedFromMirror) { - // The mirror copy only materializes branches and tags. Any other ref - // (refs/pull/*, etc.) must be fetched from the network so its local - // destination ref exists for checkout. For branches/tags, verify the - // wanted ref/commit is present and fetch it directly if not. - const upperRef = (settings.ref || '').toUpperCase(); - const coveredByMirror = !settings.ref || - !upperRef.startsWith('REFS/') || - upperRef.startsWith('REFS/HEADS/') || - upperRef.startsWith('REFS/TAGS/'); - let verified = false; - if (coveredByMirror) { - verified = yield refHelper.testRef(git, settings.ref, settings.commit); - if (verified && settings.commit) { - verified = yield git.shaExists(settings.commit); - } - } - if (!verified) { - const refSpec = refHelper.getRefSpec(settings.ref, settings.commit); - yield git.fetch(refSpec, fetchOptions); - } - } - else { - // Fetch all branches and tags - let refSpec = refHelper.getRefSpecForAllHistory(settings.ref, settings.commit); - yield git.fetch(refSpec, fetchOptions); - // When all history is fetched, the ref we're interested in may have moved to a different - // commit (push or force push). If so, fetch again with a targeted refspec. - if (!(yield refHelper.testRef(git, settings.ref, settings.commit))) { - refSpec = refHelper.getRefSpec(settings.ref, settings.commit); - yield git.fetch(refSpec, fetchOptions); - } - } - } - else { - fetchOptions.fetchDepth = settings.fetchDepth; - fetchOptions.fetchTags = settings.fetchTags; - // With the mirror attached as an alternate, git would treat every - // mirror ref as a known tip: the deepening fetch's connectivity check - // walks the whole mirror object graph and negotiation offers every - // mirror ref. Turn alternate ref discovery off and offer a few mirror - // tips explicitly instead, so the server still sends only the delta - // from the mirror (see resolveShallowNegotiationTips). - if (cacheInfo && - (yield git.version()).checkMinimum(git_command_manager_1.MinimumGitAlternateRefsCommandVersion) && - (yield blacksmithCache.sharesMirrorObjects(settings.repositoryPath, cacheInfo.mirrorPath))) { - fetchOptions.ignoreAlternateRefs = true; - fetchOptions.negotiationTips = - yield blacksmithCache.resolveShallowNegotiationTips(cacheInfo.mirrorPath, settings.ref, process.env['GITHUB_BASE_REF'] || ''); - core.info(`[git-mirror] Shallow fetch negotiating from ${fetchOptions.negotiationTips.length} mirror tip(s) with alternate ref discovery disabled`); - } - const refSpec = refHelper.getRefSpec(settings.ref, settings.commit); - yield git.fetch(refSpec, fetchOptions); - } - if (cacheInfo) { - // Mirror-served: the fetch pulls only the delta the mirror is missing. - report.delta_fetch_ms = Date.now() - fetchStart; - const objectsBytesAfter = yield mirrorTelemetry.dirSizeBytesOrNull(objectsDir); - report.delta_fetch_bytes = - objectsBytesBefore !== null && objectsBytesAfter !== null - ? Math.max(0, objectsBytesAfter - objectsBytesBefore) - : 0; + if (gitfsRevision && cacheInfo) { + // The tree is already served from the mirror: give the workspace the + // refs and index the regular path would have produced and skip the + // fetch + checkout. + core.startGroup('Pointing the repository at the mounted revision'); + const pointStart = Date.now(); + yield blacksmithCache.fetchRefsFromMirror(settings.repositoryPath, cacheInfo.mirrorPath, git.getEnvironment()); + yield gitfs.pointWorkspaceAt(settings.repositoryPath, settings.ref, gitfsRevision, git.getEnvironment()); + report.clone_from_mirror_ms += Date.now() - pointStart; report.mirror_size_bytes = yield mirrorTelemetry.packSizeBytes(cacheInfo.mirrorPath); report.ref_count = yield mirrorTelemetry.refCount(cacheInfo.mirrorPath); - } - else { - // Fallback/bypass: the fetch is the full checkout cost. - report.full_checkout_ms = Date.now() - fetchStart; - } - core.endGroup(); - // Checkout info - core.startGroup('Determining the checkout info'); - const checkoutInfo = yield refHelper.getCheckoutInfo(git, settings.ref, settings.commit); - core.endGroup(); - // LFS fetch - // Explicit lfs-fetch to avoid slow checkout (fetches one lfs object at a time). - // Explicit lfs fetch will fetch lfs objects in parallel. - // For sparse checkouts, let `checkout` fetch the needed objects lazily. - if (settings.lfs && !settings.sparseCheckout) { - core.startGroup('Fetching LFS objects'); - const lfsStart = Date.now(); - yield git.lfsFetch(checkoutInfo.startPoint || checkoutInfo.ref); - report.lfs_ms = Date.now() - lfsStart; - core.endGroup(); - } - // Sparse checkout - if (!settings.sparseCheckout) { - let gitVersion = yield git.version(); - // no need to disable sparse-checkout if the installed git runtime doesn't even support it. - if (gitVersion.checkMinimum(git_command_manager_1.MinimumGitSparseCheckoutVersion)) { - yield git.disableSparseCheckout(); - } - } - else { - core.startGroup('Setting up sparse checkout'); - if (settings.sparseCheckoutConeMode) { - yield git.sparseCheckout(settings.sparseCheckout); - } - else { - yield git.sparseCheckoutNonConeMode(settings.sparseCheckout); - } core.endGroup(); } - // Checkout - core.startGroup('Checking out the ref'); - const checkoutStart = Date.now(); - yield git.checkout(checkoutInfo.ref, checkoutInfo.startPoint); - if (cacheInfo) { - // Materializing the working tree reads objects from the mirror through - // alternates — the second half of the clone-from-mirror cost. - report.clone_from_mirror_ms += Date.now() - checkoutStart; - } else { - // Fallback/bypass: working-tree materialization is part of the full - // checkout cost, keeping phase totals comparable across serving modes. - report.full_checkout_ms += Date.now() - checkoutStart; - } - core.endGroup(); - // Dissociate from Blacksmith mirror if requested - // This copies all objects from alternates into the local repo so it's independent - if (settings.dissociate && cacheInfo) { - core.startGroup('Dissociating from Blacksmith mirror'); - yield blacksmithCache.dissociate(settings.repositoryPath, git.getEnvironment()); - core.endGroup(); + yield fetchAndCheckout(settings, git, cacheInfo, mirrorFresh, report); } // Submodules if (settings.submodules) { @@ -4048,6 +3931,210 @@ function getSourceInner(settings, report, onHydrationReport) { } }); } +/** + * Mount the requested revision from the mirror with gitfs. Returns null, + * after logging why, when this checkout is outside the gitfs cohort; throws + * when gitfs was attempted and failed. + */ +function mountWithGitfs(settings, cacheInfo, repositoryUrl) { + return __awaiter(this, void 0, void 0, function* () { + var _a; + const reason = gitfs.gitfsIneligibilityReason(settings, { + mirrorPath: (_a = cacheInfo === null || cacheInfo === void 0 ? void 0 : cacheInfo.mirrorPath) !== null && _a !== void 0 ? _a : null + }); + if (reason || !cacheInfo) { + core.info(`[gitfs] not used: ${reason}`); + return null; + } + const binary = yield gitfs.findBinary(); + if (!binary) { + core.info('[gitfs] not used: gitfs binary not found on PATH'); + return null; + } + const candidates = gitfs.mirrorRevisionCandidates(settings.ref, settings.commit); + let revision = yield gitfs.resolveInMirror(cacheInfo.mirrorPath, candidates); + if (!revision) { + // The mirror does not have the requested revision yet (a push it has + // not synced, a pull-request merge ref): fetch just that into the + // mirror, where the regular path would have fetched it into the + // workspace. The mirror changed, so the post step commits it. + const refspecs = gitfs.mirrorFetchRefspecs(settings.ref, settings.commit); + core.info(`[gitfs] ${settings.commit || settings.ref} is not in the mirror, fetching ${refspecs.join(' ')}`); + yield gitfs.fetchIntoMirror(cacheInfo.mirrorPath, repositoryUrl, settings.authToken, refspecs); + stateHelper.setBlacksmithCacheMirrorChanged(true); + revision = yield gitfs.resolveInMirror(cacheInfo.mirrorPath, candidates); + } + if (!revision) { + core.info(`[gitfs] not used: ${settings.commit || settings.ref} could not be resolved in the mirror`); + return null; + } + const mountStart = Date.now(); + yield gitfs.mount(binary, cacheInfo.mirrorPath, revision.sha, settings.repositoryPath); + stateHelper.setGitfsMountPoint(settings.repositoryPath); + core.info(`[gitfs] ${settings.repositoryPath} serves ${revision.sha} from the mirror (mounted in ${Date.now() - mountStart} ms); the tree is written out in the background`); + return revision; + }); +} +/** + * The regular path: fetch the requested revision into the workspace (from + * the mirror where possible) and check it out. + */ +function fetchAndCheckout(settings, git, cacheInfo, mirrorFresh, report) { + return __awaiter(this, void 0, void 0, function* () { + // Fetch + core.startGroup('Fetching the repository'); + const objectsDir = path.join(settings.repositoryPath, '.git', 'objects'); + const objectsBytesBefore = cacheInfo + ? yield mirrorTelemetry.dirSizeBytesOrNull(objectsDir) + : null; + const fetchStart = Date.now(); + const fetchOptions = {}; + if (settings.filter) { + fetchOptions.filter = settings.filter; + } + else if (settings.sparseCheckout) { + fetchOptions.filter = 'blob:none'; + } + if (settings.fetchDepth <= 0) { + // When the Blacksmith mirror is available and synced with the remote, + // copy branch/tag refs from it locally (objects are already shared via + // alternates) instead of doing a full +refs/heads/* fetch over the + // network, and only ask the network for the specific ref/commit this + // run needs. + let fetchedFromMirror = false; + if (cacheInfo && + mirrorFresh && + !fetchOptions.filter && + !fsHelper.fileExistsSync(path.join(settings.repositoryPath, '.git', 'shallow'))) { + fetchedFromMirror = yield blacksmithCache.fetchRefsFromMirror(settings.repositoryPath, cacheInfo.mirrorPath, git.getEnvironment()); + } + if (fetchedFromMirror) { + // The mirror copy only materializes branches and tags. Any other ref + // (refs/pull/*, etc.) must be fetched from the network so its local + // destination ref exists for checkout. For branches/tags, verify the + // wanted ref/commit is present and fetch it directly if not. + const upperRef = (settings.ref || '').toUpperCase(); + const coveredByMirror = !settings.ref || + !upperRef.startsWith('REFS/') || + upperRef.startsWith('REFS/HEADS/') || + upperRef.startsWith('REFS/TAGS/'); + let verified = false; + if (coveredByMirror) { + verified = yield refHelper.testRef(git, settings.ref, settings.commit); + if (verified && settings.commit) { + verified = yield git.shaExists(settings.commit); + } + } + if (!verified) { + const refSpec = refHelper.getRefSpec(settings.ref, settings.commit); + yield git.fetch(refSpec, fetchOptions); + } + } + else { + // Fetch all branches and tags + let refSpec = refHelper.getRefSpecForAllHistory(settings.ref, settings.commit); + yield git.fetch(refSpec, fetchOptions); + // When all history is fetched, the ref we're interested in may have moved to a different + // commit (push or force push). If so, fetch again with a targeted refspec. + if (!(yield refHelper.testRef(git, settings.ref, settings.commit))) { + refSpec = refHelper.getRefSpec(settings.ref, settings.commit); + yield git.fetch(refSpec, fetchOptions); + } + } + } + else { + fetchOptions.fetchDepth = settings.fetchDepth; + fetchOptions.fetchTags = settings.fetchTags; + // With the mirror attached as an alternate, git would treat every + // mirror ref as a known tip: the deepening fetch's connectivity check + // walks the whole mirror object graph and negotiation offers every + // mirror ref. Turn alternate ref discovery off and offer a few mirror + // tips explicitly instead, so the server still sends only the delta + // from the mirror (see resolveShallowNegotiationTips). + if (cacheInfo && + (yield git.version()).checkMinimum(git_command_manager_1.MinimumGitAlternateRefsCommandVersion) && + (yield blacksmithCache.sharesMirrorObjects(settings.repositoryPath, cacheInfo.mirrorPath))) { + fetchOptions.ignoreAlternateRefs = true; + fetchOptions.negotiationTips = + yield blacksmithCache.resolveShallowNegotiationTips(cacheInfo.mirrorPath, settings.ref, process.env['GITHUB_BASE_REF'] || ''); + core.info(`[git-mirror] Shallow fetch negotiating from ${fetchOptions.negotiationTips.length} mirror tip(s) with alternate ref discovery disabled`); + } + const refSpec = refHelper.getRefSpec(settings.ref, settings.commit); + yield git.fetch(refSpec, fetchOptions); + } + if (cacheInfo) { + // Mirror-served: the fetch pulls only the delta the mirror is missing. + report.delta_fetch_ms = Date.now() - fetchStart; + const objectsBytesAfter = yield mirrorTelemetry.dirSizeBytesOrNull(objectsDir); + report.delta_fetch_bytes = + objectsBytesBefore !== null && objectsBytesAfter !== null + ? Math.max(0, objectsBytesAfter - objectsBytesBefore) + : 0; + report.mirror_size_bytes = yield mirrorTelemetry.packSizeBytes(cacheInfo.mirrorPath); + report.ref_count = yield mirrorTelemetry.refCount(cacheInfo.mirrorPath); + } + else { + // Fallback/bypass: the fetch is the full checkout cost. + report.full_checkout_ms = Date.now() - fetchStart; + } + core.endGroup(); + // Checkout info + core.startGroup('Determining the checkout info'); + const checkoutInfo = yield refHelper.getCheckoutInfo(git, settings.ref, settings.commit); + core.endGroup(); + // LFS fetch + // Explicit lfs-fetch to avoid slow checkout (fetches one lfs object at a time). + // Explicit lfs fetch will fetch lfs objects in parallel. + // For sparse checkouts, let `checkout` fetch the needed objects lazily. + if (settings.lfs && !settings.sparseCheckout) { + core.startGroup('Fetching LFS objects'); + const lfsStart = Date.now(); + yield git.lfsFetch(checkoutInfo.startPoint || checkoutInfo.ref); + report.lfs_ms = Date.now() - lfsStart; + core.endGroup(); + } + // Sparse checkout + if (!settings.sparseCheckout) { + let gitVersion = yield git.version(); + // no need to disable sparse-checkout if the installed git runtime doesn't even support it. + if (gitVersion.checkMinimum(git_command_manager_1.MinimumGitSparseCheckoutVersion)) { + yield git.disableSparseCheckout(); + } + } + else { + core.startGroup('Setting up sparse checkout'); + if (settings.sparseCheckoutConeMode) { + yield git.sparseCheckout(settings.sparseCheckout); + } + else { + yield git.sparseCheckoutNonConeMode(settings.sparseCheckout); + } + core.endGroup(); + } + // Checkout + core.startGroup('Checking out the ref'); + const checkoutStart = Date.now(); + yield git.checkout(checkoutInfo.ref, checkoutInfo.startPoint); + if (cacheInfo) { + // Materializing the working tree reads objects from the mirror through + // alternates — the second half of the clone-from-mirror cost. + report.clone_from_mirror_ms += Date.now() - checkoutStart; + } + else { + // Fallback/bypass: working-tree materialization is part of the full + // checkout cost, keeping phase totals comparable across serving modes. + report.full_checkout_ms += Date.now() - checkoutStart; + } + core.endGroup(); + // Dissociate from Blacksmith mirror if requested + // This copies all objects from alternates into the local repo so it's independent + if (settings.dissociate && cacheInfo) { + core.startGroup('Dissociating from Blacksmith mirror'); + yield blacksmithCache.dissociate(settings.repositoryPath, git.getEnvironment()); + core.endGroup(); + } + }); +} function cleanup(repositoryPath) { return __awaiter(this, void 0, void 0, function* () { // Repo exists? @@ -4181,6 +4268,353 @@ class GitVersion { exports.GitVersion = GitVersion; +/***/ }), + +/***/ 1574: +/***/ (function(__unused_webpack_module, exports, __nccwpck_require__) { + +"use strict"; + +var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) { + if (k2 === undefined) k2 = k; + var desc = Object.getOwnPropertyDescriptor(m, k); + if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) { + desc = { enumerable: true, get: function() { return m[k]; } }; + } + Object.defineProperty(o, k2, desc); +}) : (function(o, m, k, k2) { + if (k2 === undefined) k2 = k; + o[k2] = m[k]; +})); +var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) { + Object.defineProperty(o, "default", { enumerable: true, value: v }); +}) : function(o, v) { + o["default"] = v; +}); +var __importStar = (this && this.__importStar) || function (mod) { + if (mod && mod.__esModule) return mod; + var result = {}; + if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k); + __setModuleDefault(result, mod); + return result; +}; +var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) { + function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); } + return new (P || (P = Promise))(function (resolve, reject) { + function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } } + function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } } + function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); } + step((generator = generator.apply(thisArg, _arguments || [])).next()); + }); +}; +Object.defineProperty(exports, "__esModule", ({ value: true })); +exports.GITFS_ENV = void 0; +exports.isGitfsRequested = isGitfsRequested; +exports.gitfsIneligibilityReason = gitfsIneligibilityReason; +exports.findBinary = findBinary; +exports.mirrorRevisionCandidates = mirrorRevisionCandidates; +exports.mirrorFetchRefspecs = mirrorFetchRefspecs; +exports.resolveInMirror = resolveInMirror; +exports.fetchIntoMirror = fetchIntoMirror; +exports.mount = mount; +exports.pointWorkspaceAt = pointWorkspaceAt; +exports.teardown = teardown; +const core = __importStar(__nccwpck_require__(2186)); +const exec = __importStar(__nccwpck_require__(1514)); +const fs = __importStar(__nccwpck_require__(7147)); +const io = __importStar(__nccwpck_require__(7436)); +const path = __importStar(__nccwpck_require__(1017)); +const container_detector_1 = __nccwpck_require__(6424); +const blacksmith_cache_1 = __nccwpck_require__(9242); +/** + * gitfs serves the workspace from the git mirror sticky disk instead of + * checking it out: `gitfs mount --ref --daemon` + * returns as soon as the tree is mounted (milliseconds), writes the real + * checkout out in the background and swaps it in for the mount once it is + * complete. Writes made through the mount in the meantime (`.git`, npm's + * node_modules, ...) land in that same tree. The agent enables it per job via + * BLACKSMITH_USE_GITFS=true, driven by the installation's use_gitfs_checkout + * flag; anything gitfs cannot serve falls through to the regular checkout. + */ +exports.GITFS_ENV = 'BLACKSMITH_USE_GITFS'; +const GITFS_BINARY = 'gitfs'; +const DAEMON_EXIT_TIMEOUT_MS = 10000; +const DAEMON_KILL_TIMEOUT_MS = 5000; +function isGitfsRequested(env = process.env) { + return env[exports.GITFS_ENV] === 'true'; +} +/** + * Why gitfs cannot serve this checkout, or null when it can. The v1 cohort is + * the plain case: a Linux runner VM with the mirror mounted, a branch, tag, + * pull ref or commit to serve, and none of the features that need the + * workspace to be a real fetch (submodules, LFS, sparse/partial clones, + * dissociate). + */ +function gitfsIneligibilityReason(settings, ctx) { + var _a, _b; + const platform = (_a = ctx.platform) !== null && _a !== void 0 ? _a : process.platform; + if (platform !== 'linux') { + return `platform ${platform} is not linux`; + } + if (!ctx.mirrorPath) { + return 'git mirror is not available'; + } + const inContainer = (_b = ctx.inContainer) !== null && _b !== void 0 ? _b : (0, container_detector_1.isRunningInContainer)(); + if (inContainer) { + return 'running inside a container'; + } + if (!settings.ref && !settings.commit) { + return 'no ref or commit requested'; + } + if (settings.submodules) { + return 'submodules are enabled'; + } + if (settings.lfs) { + return 'lfs is enabled'; + } + if (settings.sparseCheckout && settings.sparseCheckout.length > 0) { + return 'sparse checkout is enabled'; + } + if (settings.filter) { + return `filter ${settings.filter} is set`; + } + if (settings.dissociate) { + return 'dissociate is enabled'; + } + if (fs.existsSync(path.join(settings.repositoryPath, '.git'))) { + return 'workspace already contains a repository'; + } + return null; +} +function findBinary() { + return __awaiter(this, void 0, void 0, function* () { + const found = yield io.which(GITFS_BINARY, false); + return found || null; + }); +} +function classify(ref) { + const upper = ref.toUpperCase(); + if (upper.startsWith('REFS/HEADS/')) + return 'branch'; + if (upper.startsWith('REFS/TAGS/')) + return 'tag'; + if (upper.startsWith('REFS/PULL/')) + return 'pull'; + return 'other'; +} +/** + * Revisions to try in the mirror for the requested ref/commit, in order. A + * commit wins over the ref (that is what the checkout would end up at); an + * unqualified ref is a branch first, then a tag, as in getCheckoutInfo. + */ +function mirrorRevisionCandidates(ref, commit) { + if (commit) { + const kind = ref ? classify(ref) : 'commit'; + const mirrorRef = ref && ref.toUpperCase().startsWith('REFS/') ? ref : ''; + return [{ rev: `${commit}^{commit}`, mirrorRef, kind }]; + } + if (!ref) { + return []; + } + if (ref.toUpperCase().startsWith('REFS/')) { + return [{ rev: `${ref}^{commit}`, mirrorRef: ref, kind: classify(ref) }]; + } + return [ + { + rev: `refs/heads/${ref}^{commit}`, + mirrorRef: `refs/heads/${ref}`, + kind: 'branch' + }, + { + rev: `refs/tags/${ref}^{commit}`, + mirrorRef: `refs/tags/${ref}`, + kind: 'tag' + } + ]; +} +/** + * Refspecs that bring the requested ref/commit into the mirror when it is + * not there yet (a push the mirror has not synced, or a pull-request merge + * ref). Qualified refs are fetched under their own name so the workspace + * can mirror them; a bare commit is fetched by sha. + */ +function mirrorFetchRefspecs(ref, commit) { + const qualified = !!ref && ref.toUpperCase().startsWith('REFS/'); + if (commit) { + return qualified ? [`+${commit}:${ref}`] : [commit]; + } + if (qualified) { + return [`+${ref}:${ref}`]; + } + return [ + `+refs/heads/${ref}*:refs/heads/${ref}*`, + `+refs/tags/${ref}*:refs/tags/${ref}*` + ]; +} +function resolveInMirror(mirrorPath, candidates) { + return __awaiter(this, void 0, void 0, function* () { + for (const candidate of candidates) { + const result = yield exec.getExecOutput('git', ['-C', mirrorPath, 'rev-parse', '--verify', '--quiet', candidate.rev], { silent: true, ignoreReturnCode: true }); + const sha = result.stdout.trim(); + if (result.exitCode === 0 && /^[0-9a-f]{40}$/.test(sha)) { + return { sha, mirrorRef: candidate.mirrorRef, kind: candidate.kind }; + } + } + return null; + }); +} +function fetchIntoMirror(mirrorPath, repositoryUrl, authToken, refspecs) { + return __awaiter(this, void 0, void 0, function* () { + const auth = (0, blacksmith_cache_1.getAuthConfigArgs)(repositoryUrl, authToken); + yield exec.exec('git', [ + '-C', + mirrorPath, + '-c', + `${auth.configKey}=${auth.configValue}`, + '-c', + 'gc.auto=0', + 'fetch', + '--no-tags', + '--no-recurse-submodules', + '--no-write-fetch-head', + repositoryUrl, + ...refspecs + ]); + }); +} +function mount(binary, mirrorPath, sha, mountpoint) { + return __awaiter(this, void 0, void 0, function* () { + if (fs.existsSync(mountpoint) && fs.readdirSync(mountpoint).length > 0) { + throw new Error(`${mountpoint} is not empty`); + } + yield exec.exec(binary, [ + 'mount', + mirrorPath, + '--ref', + sha, + mountpoint, + '--daemon' + ]); + }); +} +/** + * Point the workspace at the mounted revision without touching the tree: + * the local branch / tag / pull ref the regular checkout would have created, + * HEAD on it (or detached, as `git checkout` would leave it), and an index + * for the tree. The files are already there, served by gitfs. + */ +function pointWorkspaceAt(repositoryPath, ref, revision, env) { + return __awaiter(this, void 0, void 0, function* () { + const git = (...args) => __awaiter(this, void 0, void 0, function* () { + yield exec.exec('git', ['-C', repositoryPath, ...args], { env }); + }); + const { sha } = revision; + if (revision.kind === 'branch') { + const branch = revision.mirrorRef.slice('refs/heads/'.length); + yield git('update-ref', `refs/remotes/origin/${branch}`, sha); + yield git('update-ref', `refs/heads/${branch}`, sha); + yield git('config', `branch.${branch}.remote`, 'origin'); + yield git('config', `branch.${branch}.merge`, `refs/heads/${branch}`); + yield git('symbolic-ref', 'HEAD', `refs/heads/${branch}`); + } + else { + if (revision.kind === 'tag') { + yield git('update-ref', revision.mirrorRef, sha); + } + else if (revision.kind === 'pull') { + const name = revision.mirrorRef.slice('refs/pull/'.length); + yield git('update-ref', `refs/remotes/pull/${name}`, sha); + } + yield git('update-ref', '--no-deref', 'HEAD', sha); + } + // Index entries without stat data: git re-verifies file contents on the + // first status/diff instead of touching every file now. + yield git('read-tree', sha); + }); +} +function daemonPids(mirrorPath) { + const pids = []; + let entries; + try { + entries = fs.readdirSync('/proc'); + } + catch (_a) { + return pids; + } + for (const entry of entries) { + if (!/^\d+$/.test(entry)) + continue; + let cmdline; + try { + cmdline = fs.readFileSync(`/proc/${entry}/cmdline`, 'utf8'); + } + catch (_b) { + continue; + } + const argv = cmdline.split('\0'); + if (path.basename(argv[0] || '') === GITFS_BINARY && + argv.includes('mount') && + argv.includes(mirrorPath)) { + pids.push(Number(entry)); + } + } + return pids; +} +function waitForDaemons(mirrorPath, timeoutMs) { + return __awaiter(this, void 0, void 0, function* () { + const deadline = Date.now() + timeoutMs; + let pids = daemonPids(mirrorPath); + while (pids.length > 0 && Date.now() < deadline) { + yield new Promise(resolve => setTimeout(resolve, 100)); + pids = daemonPids(mirrorPath); + } + return pids; + }); +} +/** + * Take gitfs down before the mirror sticky disk is unmounted: the daemon has + * the mirror's packs mapped, so the disk cannot be released while it lives. + * While the tree is still being written the workspace path is a symlink into + * the mount; unmounting then discards the partial tree and leaves the + * workspace empty. After the swap the path is a plain directory and the + * daemon exits on its own once nothing holds the old mount open. + */ +function teardown(mountpoint, mirrorPath) { + return __awaiter(this, void 0, void 0, function* () { + let isSymlink = false; + try { + isSymlink = fs.lstatSync(mountpoint).isSymbolicLink(); + } + catch (_a) { + // mountpoint gone + } + if (isSymlink) { + const binary = yield findBinary(); + if (binary) { + core.info(`[gitfs] Unmounting ${mountpoint}`); + yield exec.exec(binary, ['unmount', mountpoint], { ignoreReturnCode: true }); + } + } + let pids = yield waitForDaemons(mirrorPath, DAEMON_EXIT_TIMEOUT_MS); + if (pids.length > 0) { + core.info(`[gitfs] Stopping daemon(s) ${pids.join(', ')} still serving ${mirrorPath}`); + for (const pid of pids) { + try { + process.kill(pid, 'SIGTERM'); + } + catch (_b) { + // already gone + } + } + pids = yield waitForDaemons(mirrorPath, DAEMON_KILL_TIMEOUT_MS); + if (pids.length > 0) { + core.warning(`[gitfs] daemon(s) ${pids.join(', ')} did not exit; the mirror unmount may fail`); + } + } + }); +} + + /***/ }), /***/ 138: @@ -4691,6 +5125,7 @@ const inputHelper = __importStar(__nccwpck_require__(5480)); const path = __importStar(__nccwpck_require__(1017)); const stateHelper = __importStar(__nccwpck_require__(4866)); const blacksmithCache = __importStar(__nccwpck_require__(9242)); +const gitfs = __importStar(__nccwpck_require__(1574)); const step_checker_1 = __nccwpck_require__(9716); const internal_metrics_1 = __nccwpck_require__(7753); const mirrorTelemetry = __importStar(__nccwpck_require__(7185)); @@ -4731,6 +5166,16 @@ function cleanup() { const repoName = stateHelper.BlacksmithCacheRepoName; const mountPoint = stateHelper.BlacksmithCacheMountPoint; const mirrorPath = stateHelper.BlacksmithCacheMirrorPath; + // gitfs maps the mirror's packs, so it has to be gone before the sticky + // disk can be unmounted below. + if (stateHelper.GitfsMountPoint && mirrorPath) { + try { + yield gitfs.teardown(stateHelper.GitfsMountPoint, mirrorPath); + } + catch (error) { + core.warning(`[gitfs] teardown failed: ${error}`); + } + } const performedHydration = stateHelper.BlacksmithCachePerformedHydration; let mirrorChanged = stateHelper.BlacksmithCacheMirrorChanged; let mirrorSyncFailed = stateHelper.BlacksmithCacheMirrorSyncFailed; @@ -5524,7 +5969,7 @@ var __importStar = (this && this.__importStar) || function (mod) { return result; }; Object.defineProperty(exports, "__esModule", ({ value: true })); -exports.BlacksmithCacheCommitEarlyDenyReason = exports.BlacksmithCacheVerbose = exports.BlacksmithCacheRepoUrl = exports.BlacksmithCacheMirrorSyncDeferred = exports.BlacksmithCacheMirrorSyncTimedOut = exports.BlacksmithCacheMirrorSyncFailed = exports.BlacksmithCacheMirrorChanged = exports.BlacksmithCachePerformedHydration = exports.BlacksmithCacheStickyDiskKey = exports.BlacksmithCacheRepoName = exports.BlacksmithCacheMountPoint = exports.BlacksmithCacheMirrorPath = exports.BlacksmithCacheExposeId = exports.SshKnownHostsPath = exports.SshKeyPath = exports.PostSetSafeDirectory = exports.RepositoryPath = exports.IsPost = void 0; +exports.BlacksmithCacheCommitEarlyDenyReason = exports.GitfsMountPoint = exports.BlacksmithCacheVerbose = exports.BlacksmithCacheRepoUrl = exports.BlacksmithCacheMirrorSyncDeferred = exports.BlacksmithCacheMirrorSyncTimedOut = exports.BlacksmithCacheMirrorSyncFailed = exports.BlacksmithCacheMirrorChanged = exports.BlacksmithCachePerformedHydration = exports.BlacksmithCacheStickyDiskKey = exports.BlacksmithCacheRepoName = exports.BlacksmithCacheMountPoint = exports.BlacksmithCacheMirrorPath = exports.BlacksmithCacheExposeId = exports.SshKnownHostsPath = exports.SshKeyPath = exports.PostSetSafeDirectory = exports.RepositoryPath = exports.IsPost = void 0; exports.setRepositoryPath = setRepositoryPath; exports.setSshKeyPath = setSshKeyPath; exports.setSshKnownHostsPath = setSshKnownHostsPath; @@ -5542,6 +5987,7 @@ exports.setBlacksmithCacheMirrorSyncDeferred = setBlacksmithCacheMirrorSyncDefer exports.setBlacksmithCacheRepoUrl = setBlacksmithCacheRepoUrl; exports.setBlacksmithCacheVerbose = setBlacksmithCacheVerbose; exports.setBlacksmithCacheCommitEarlyDenyReason = setBlacksmithCacheCommitEarlyDenyReason; +exports.setGitfsMountPoint = setGitfsMountPoint; const core = __importStar(__nccwpck_require__(2186)); /** * Indicates whether the POST action is running @@ -5616,6 +6062,11 @@ exports.BlacksmithCacheVerbose = core.getState('blacksmithCacheVerbose') === 'tr * Reason the host gave at mount time for denying this job's sticky disk commit * (e.g. branch protection). Empty when no denial was reported. */ +/** + * Workspace path served by gitfs, so the POST action can take the mount down + * before the mirror sticky disk is unmounted. + */ +exports.GitfsMountPoint = core.getState('gitfsMountPoint'); exports.BlacksmithCacheCommitEarlyDenyReason = core.getState('blacksmithCacheCommitEarlyDenyReason'); /** * Save the repository path so the POST action can retrieve the value. @@ -5718,6 +6169,9 @@ function setBlacksmithCacheVerbose(verbose) { function setBlacksmithCacheCommitEarlyDenyReason(reason) { core.saveState('blacksmithCacheCommitEarlyDenyReason', reason); } +function setGitfsMountPoint(mountPoint) { + core.saveState('gitfsMountPoint', mountPoint); +} // Publish a variable so that when the POST action runs, it can determine it should run the cleanup logic. // This is necessary since we don't have a separate entry point. if (!exports.IsPost) { diff --git a/src/blacksmith-cache.ts b/src/blacksmith-cache.ts index f073efc..bef72d6 100644 --- a/src/blacksmith-cache.ts +++ b/src/blacksmith-cache.ts @@ -431,7 +431,7 @@ export async function setupCache( * 1. The header value is not visible in process arguments * 2. It follows the same pattern used by the upstream checkout action */ -function getAuthConfigArgs( +export function getAuthConfigArgs( repoUrl: string, authToken: string ): {configKey: string; configValue: string} { diff --git a/src/git-source-provider.ts b/src/git-source-provider.ts index f4c9d45..83de3bd 100644 --- a/src/git-source-provider.ts +++ b/src/git-source-provider.ts @@ -11,6 +11,7 @@ import * as stateHelper from './state-helper' import * as urlHelper from './url-helper' import * as blacksmithCache from './blacksmith-cache' import * as mirrorTelemetry from './mirror-telemetry' +import * as gitfs from './gitfs' import { FetchOptions, MinimumGitAlternateRefsCommandVersion, @@ -295,6 +296,25 @@ async function getSourceInner( } } + // Serve the workspace with gitfs when the job asks for it: mount the + // requested revision from the mirror now (milliseconds), and let the + // tree be written out behind the job. The .git set up below is created + // through the mount, so git works from the first step. Anything gitfs + // cannot serve falls through to the regular fetch + checkout. + let gitfsRevision: gitfs.MirrorRevision | null = null + if (gitfs.isGitfsRequested()) { + core.startGroup('Mounting the repository with gitfs') + try { + gitfsRevision = await mountWithGitfs(settings, cacheInfo, repositoryUrl) + } catch (error) { + core.warning( + `[gitfs] mount failed, using the regular checkout: ${error}` + ) + gitfsRevision = null + } + core.endGroup() + } + // Initialize the repository if ( !fsHelper.directoryExistsSync(path.join(settings.repositoryPath, '.git')) @@ -355,194 +375,31 @@ async function getSourceInner( await git.lfsInstall() } - // Fetch - core.startGroup('Fetching the repository') - const objectsDir = path.join(settings.repositoryPath, '.git', 'objects') - const objectsBytesBefore = cacheInfo - ? await mirrorTelemetry.dirSizeBytesOrNull(objectsDir) - : null - const fetchStart = Date.now() - const fetchOptions: FetchOptions = {} - - if (settings.filter) { - fetchOptions.filter = settings.filter - } else if (settings.sparseCheckout) { - fetchOptions.filter = 'blob:none' - } - - if (settings.fetchDepth <= 0) { - // When the Blacksmith mirror is available and synced with the remote, - // copy branch/tag refs from it locally (objects are already shared via - // alternates) instead of doing a full +refs/heads/* fetch over the - // network, and only ask the network for the specific ref/commit this - // run needs. - let fetchedFromMirror = false - if ( - cacheInfo && - mirrorFresh && - !fetchOptions.filter && - !fsHelper.fileExistsSync( - path.join(settings.repositoryPath, '.git', 'shallow') - ) - ) { - fetchedFromMirror = await blacksmithCache.fetchRefsFromMirror( - settings.repositoryPath, - cacheInfo.mirrorPath, - git.getEnvironment() - ) - } - - if (fetchedFromMirror) { - // The mirror copy only materializes branches and tags. Any other ref - // (refs/pull/*, etc.) must be fetched from the network so its local - // destination ref exists for checkout. For branches/tags, verify the - // wanted ref/commit is present and fetch it directly if not. - const upperRef = (settings.ref || '').toUpperCase() - const coveredByMirror = - !settings.ref || - !upperRef.startsWith('REFS/') || - upperRef.startsWith('REFS/HEADS/') || - upperRef.startsWith('REFS/TAGS/') - let verified = false - if (coveredByMirror) { - verified = await refHelper.testRef(git, settings.ref, settings.commit) - if (verified && settings.commit) { - verified = await git.shaExists(settings.commit) - } - } - if (!verified) { - const refSpec = refHelper.getRefSpec(settings.ref, settings.commit) - await git.fetch(refSpec, fetchOptions) - } - } else { - // Fetch all branches and tags - let refSpec = refHelper.getRefSpecForAllHistory( - settings.ref, - settings.commit - ) - await git.fetch(refSpec, fetchOptions) - - // When all history is fetched, the ref we're interested in may have moved to a different - // commit (push or force push). If so, fetch again with a targeted refspec. - if (!(await refHelper.testRef(git, settings.ref, settings.commit))) { - refSpec = refHelper.getRefSpec(settings.ref, settings.commit) - await git.fetch(refSpec, fetchOptions) - } - } - } else { - fetchOptions.fetchDepth = settings.fetchDepth - fetchOptions.fetchTags = settings.fetchTags - // With the mirror attached as an alternate, git would treat every - // mirror ref as a known tip: the deepening fetch's connectivity check - // walks the whole mirror object graph and negotiation offers every - // mirror ref. Turn alternate ref discovery off and offer a few mirror - // tips explicitly instead, so the server still sends only the delta - // from the mirror (see resolveShallowNegotiationTips). - if ( - cacheInfo && - (await git.version()).checkMinimum( - MinimumGitAlternateRefsCommandVersion - ) && - (await blacksmithCache.sharesMirrorObjects( - settings.repositoryPath, - cacheInfo.mirrorPath - )) - ) { - fetchOptions.ignoreAlternateRefs = true - fetchOptions.negotiationTips = - await blacksmithCache.resolveShallowNegotiationTips( - cacheInfo.mirrorPath, - settings.ref, - process.env['GITHUB_BASE_REF'] || '' - ) - core.info( - `[git-mirror] Shallow fetch negotiating from ${fetchOptions.negotiationTips.length} mirror tip(s) with alternate ref discovery disabled` - ) - } - const refSpec = refHelper.getRefSpec(settings.ref, settings.commit) - await git.fetch(refSpec, fetchOptions) - } - if (cacheInfo) { - // Mirror-served: the fetch pulls only the delta the mirror is missing. - report.delta_fetch_ms = Date.now() - fetchStart - const objectsBytesAfter = - await mirrorTelemetry.dirSizeBytesOrNull(objectsDir) - report.delta_fetch_bytes = - objectsBytesBefore !== null && objectsBytesAfter !== null - ? Math.max(0, objectsBytesAfter - objectsBytesBefore) - : 0 + if (gitfsRevision && cacheInfo) { + // The tree is already served from the mirror: give the workspace the + // refs and index the regular path would have produced and skip the + // fetch + checkout. + core.startGroup('Pointing the repository at the mounted revision') + const pointStart = Date.now() + await blacksmithCache.fetchRefsFromMirror( + settings.repositoryPath, + cacheInfo.mirrorPath, + git.getEnvironment() + ) + await gitfs.pointWorkspaceAt( + settings.repositoryPath, + settings.ref, + gitfsRevision, + git.getEnvironment() + ) + report.clone_from_mirror_ms += Date.now() - pointStart report.mirror_size_bytes = await mirrorTelemetry.packSizeBytes( cacheInfo.mirrorPath ) report.ref_count = await mirrorTelemetry.refCount(cacheInfo.mirrorPath) - } else { - // Fallback/bypass: the fetch is the full checkout cost. - report.full_checkout_ms = Date.now() - fetchStart - } - core.endGroup() - - // Checkout info - core.startGroup('Determining the checkout info') - const checkoutInfo = await refHelper.getCheckoutInfo( - git, - settings.ref, - settings.commit - ) - core.endGroup() - - // LFS fetch - // Explicit lfs-fetch to avoid slow checkout (fetches one lfs object at a time). - // Explicit lfs fetch will fetch lfs objects in parallel. - // For sparse checkouts, let `checkout` fetch the needed objects lazily. - if (settings.lfs && !settings.sparseCheckout) { - core.startGroup('Fetching LFS objects') - const lfsStart = Date.now() - await git.lfsFetch(checkoutInfo.startPoint || checkoutInfo.ref) - report.lfs_ms = Date.now() - lfsStart - core.endGroup() - } - - // Sparse checkout - if (!settings.sparseCheckout) { - let gitVersion = await git.version() - // no need to disable sparse-checkout if the installed git runtime doesn't even support it. - if (gitVersion.checkMinimum(MinimumGitSparseCheckoutVersion)) { - await git.disableSparseCheckout() - } - } else { - core.startGroup('Setting up sparse checkout') - if (settings.sparseCheckoutConeMode) { - await git.sparseCheckout(settings.sparseCheckout) - } else { - await git.sparseCheckoutNonConeMode(settings.sparseCheckout) - } core.endGroup() - } - - // Checkout - core.startGroup('Checking out the ref') - const checkoutStart = Date.now() - await git.checkout(checkoutInfo.ref, checkoutInfo.startPoint) - if (cacheInfo) { - // Materializing the working tree reads objects from the mirror through - // alternates — the second half of the clone-from-mirror cost. - report.clone_from_mirror_ms += Date.now() - checkoutStart } else { - // Fallback/bypass: working-tree materialization is part of the full - // checkout cost, keeping phase totals comparable across serving modes. - report.full_checkout_ms += Date.now() - checkoutStart - } - core.endGroup() - - // Dissociate from Blacksmith mirror if requested - // This copies all objects from alternates into the local repo so it's independent - if (settings.dissociate && cacheInfo) { - core.startGroup('Dissociating from Blacksmith mirror') - await blacksmithCache.dissociate( - settings.repositoryPath, - git.getEnvironment() - ) - core.endGroup() + await fetchAndCheckout(settings, git, cacheInfo, mirrorFresh, report) } // Submodules @@ -602,6 +459,275 @@ async function getSourceInner( } } +/** + * Mount the requested revision from the mirror with gitfs. Returns null, + * after logging why, when this checkout is outside the gitfs cohort; throws + * when gitfs was attempted and failed. + */ +async function mountWithGitfs( + settings: IGitSourceSettings, + cacheInfo: blacksmithCache.CacheInfo | null, + repositoryUrl: string +): Promise { + const reason = gitfs.gitfsIneligibilityReason(settings, { + mirrorPath: cacheInfo?.mirrorPath ?? null + }) + if (reason || !cacheInfo) { + core.info(`[gitfs] not used: ${reason}`) + return null + } + const binary = await gitfs.findBinary() + if (!binary) { + core.info('[gitfs] not used: gitfs binary not found on PATH') + return null + } + + const candidates = gitfs.mirrorRevisionCandidates( + settings.ref, + settings.commit + ) + let revision = await gitfs.resolveInMirror(cacheInfo.mirrorPath, candidates) + if (!revision) { + // The mirror does not have the requested revision yet (a push it has + // not synced, a pull-request merge ref): fetch just that into the + // mirror, where the regular path would have fetched it into the + // workspace. The mirror changed, so the post step commits it. + const refspecs = gitfs.mirrorFetchRefspecs(settings.ref, settings.commit) + core.info( + `[gitfs] ${settings.commit || settings.ref} is not in the mirror, fetching ${refspecs.join(' ')}` + ) + await gitfs.fetchIntoMirror( + cacheInfo.mirrorPath, + repositoryUrl, + settings.authToken, + refspecs + ) + stateHelper.setBlacksmithCacheMirrorChanged(true) + revision = await gitfs.resolveInMirror(cacheInfo.mirrorPath, candidates) + } + if (!revision) { + core.info( + `[gitfs] not used: ${settings.commit || settings.ref} could not be resolved in the mirror` + ) + return null + } + + const mountStart = Date.now() + await gitfs.mount( + binary, + cacheInfo.mirrorPath, + revision.sha, + settings.repositoryPath + ) + stateHelper.setGitfsMountPoint(settings.repositoryPath) + core.info( + `[gitfs] ${settings.repositoryPath} serves ${revision.sha} from the mirror (mounted in ${Date.now() - mountStart} ms); the tree is written out in the background` + ) + return revision +} + +/** + * The regular path: fetch the requested revision into the workspace (from + * the mirror where possible) and check it out. + */ +async function fetchAndCheckout( + settings: IGitSourceSettings, + git: IGitCommandManager, + cacheInfo: blacksmithCache.CacheInfo | null, + mirrorFresh: boolean, + report: mirrorTelemetry.CheckoutReport +): Promise { + // Fetch + core.startGroup('Fetching the repository') + const objectsDir = path.join(settings.repositoryPath, '.git', 'objects') + const objectsBytesBefore = cacheInfo + ? await mirrorTelemetry.dirSizeBytesOrNull(objectsDir) + : null + const fetchStart = Date.now() + const fetchOptions: FetchOptions = {} + + if (settings.filter) { + fetchOptions.filter = settings.filter + } else if (settings.sparseCheckout) { + fetchOptions.filter = 'blob:none' + } + + if (settings.fetchDepth <= 0) { + // When the Blacksmith mirror is available and synced with the remote, + // copy branch/tag refs from it locally (objects are already shared via + // alternates) instead of doing a full +refs/heads/* fetch over the + // network, and only ask the network for the specific ref/commit this + // run needs. + let fetchedFromMirror = false + if ( + cacheInfo && + mirrorFresh && + !fetchOptions.filter && + !fsHelper.fileExistsSync( + path.join(settings.repositoryPath, '.git', 'shallow') + ) + ) { + fetchedFromMirror = await blacksmithCache.fetchRefsFromMirror( + settings.repositoryPath, + cacheInfo.mirrorPath, + git.getEnvironment() + ) + } + + if (fetchedFromMirror) { + // The mirror copy only materializes branches and tags. Any other ref + // (refs/pull/*, etc.) must be fetched from the network so its local + // destination ref exists for checkout. For branches/tags, verify the + // wanted ref/commit is present and fetch it directly if not. + const upperRef = (settings.ref || '').toUpperCase() + const coveredByMirror = + !settings.ref || + !upperRef.startsWith('REFS/') || + upperRef.startsWith('REFS/HEADS/') || + upperRef.startsWith('REFS/TAGS/') + let verified = false + if (coveredByMirror) { + verified = await refHelper.testRef(git, settings.ref, settings.commit) + if (verified && settings.commit) { + verified = await git.shaExists(settings.commit) + } + } + if (!verified) { + const refSpec = refHelper.getRefSpec(settings.ref, settings.commit) + await git.fetch(refSpec, fetchOptions) + } + } else { + // Fetch all branches and tags + let refSpec = refHelper.getRefSpecForAllHistory( + settings.ref, + settings.commit + ) + await git.fetch(refSpec, fetchOptions) + + // When all history is fetched, the ref we're interested in may have moved to a different + // commit (push or force push). If so, fetch again with a targeted refspec. + if (!(await refHelper.testRef(git, settings.ref, settings.commit))) { + refSpec = refHelper.getRefSpec(settings.ref, settings.commit) + await git.fetch(refSpec, fetchOptions) + } + } + } else { + fetchOptions.fetchDepth = settings.fetchDepth + fetchOptions.fetchTags = settings.fetchTags + // With the mirror attached as an alternate, git would treat every + // mirror ref as a known tip: the deepening fetch's connectivity check + // walks the whole mirror object graph and negotiation offers every + // mirror ref. Turn alternate ref discovery off and offer a few mirror + // tips explicitly instead, so the server still sends only the delta + // from the mirror (see resolveShallowNegotiationTips). + if ( + cacheInfo && + (await git.version()).checkMinimum( + MinimumGitAlternateRefsCommandVersion + ) && + (await blacksmithCache.sharesMirrorObjects( + settings.repositoryPath, + cacheInfo.mirrorPath + )) + ) { + fetchOptions.ignoreAlternateRefs = true + fetchOptions.negotiationTips = + await blacksmithCache.resolveShallowNegotiationTips( + cacheInfo.mirrorPath, + settings.ref, + process.env['GITHUB_BASE_REF'] || '' + ) + core.info( + `[git-mirror] Shallow fetch negotiating from ${fetchOptions.negotiationTips.length} mirror tip(s) with alternate ref discovery disabled` + ) + } + const refSpec = refHelper.getRefSpec(settings.ref, settings.commit) + await git.fetch(refSpec, fetchOptions) + } + if (cacheInfo) { + // Mirror-served: the fetch pulls only the delta the mirror is missing. + report.delta_fetch_ms = Date.now() - fetchStart + const objectsBytesAfter = + await mirrorTelemetry.dirSizeBytesOrNull(objectsDir) + report.delta_fetch_bytes = + objectsBytesBefore !== null && objectsBytesAfter !== null + ? Math.max(0, objectsBytesAfter - objectsBytesBefore) + : 0 + report.mirror_size_bytes = await mirrorTelemetry.packSizeBytes( + cacheInfo.mirrorPath + ) + report.ref_count = await mirrorTelemetry.refCount(cacheInfo.mirrorPath) + } else { + // Fallback/bypass: the fetch is the full checkout cost. + report.full_checkout_ms = Date.now() - fetchStart + } + core.endGroup() + + // Checkout info + core.startGroup('Determining the checkout info') + const checkoutInfo = await refHelper.getCheckoutInfo( + git, + settings.ref, + settings.commit + ) + core.endGroup() + + // LFS fetch + // Explicit lfs-fetch to avoid slow checkout (fetches one lfs object at a time). + // Explicit lfs fetch will fetch lfs objects in parallel. + // For sparse checkouts, let `checkout` fetch the needed objects lazily. + if (settings.lfs && !settings.sparseCheckout) { + core.startGroup('Fetching LFS objects') + const lfsStart = Date.now() + await git.lfsFetch(checkoutInfo.startPoint || checkoutInfo.ref) + report.lfs_ms = Date.now() - lfsStart + core.endGroup() + } + + // Sparse checkout + if (!settings.sparseCheckout) { + let gitVersion = await git.version() + // no need to disable sparse-checkout if the installed git runtime doesn't even support it. + if (gitVersion.checkMinimum(MinimumGitSparseCheckoutVersion)) { + await git.disableSparseCheckout() + } + } else { + core.startGroup('Setting up sparse checkout') + if (settings.sparseCheckoutConeMode) { + await git.sparseCheckout(settings.sparseCheckout) + } else { + await git.sparseCheckoutNonConeMode(settings.sparseCheckout) + } + core.endGroup() + } + + // Checkout + core.startGroup('Checking out the ref') + const checkoutStart = Date.now() + await git.checkout(checkoutInfo.ref, checkoutInfo.startPoint) + if (cacheInfo) { + // Materializing the working tree reads objects from the mirror through + // alternates — the second half of the clone-from-mirror cost. + report.clone_from_mirror_ms += Date.now() - checkoutStart + } else { + // Fallback/bypass: working-tree materialization is part of the full + // checkout cost, keeping phase totals comparable across serving modes. + report.full_checkout_ms += Date.now() - checkoutStart + } + core.endGroup() + + // Dissociate from Blacksmith mirror if requested + // This copies all objects from alternates into the local repo so it's independent + if (settings.dissociate && cacheInfo) { + core.startGroup('Dissociating from Blacksmith mirror') + await blacksmithCache.dissociate( + settings.repositoryPath, + git.getEnvironment() + ) + core.endGroup() + } +} + export async function cleanup(repositoryPath: string): Promise { // Repo exists? if ( diff --git a/src/gitfs.ts b/src/gitfs.ts new file mode 100644 index 0000000..b8245ce --- /dev/null +++ b/src/gitfs.ts @@ -0,0 +1,357 @@ +import * as core from '@actions/core' +import * as exec from '@actions/exec' +import * as fs from 'fs' +import * as io from '@actions/io' +import * as path from 'path' +import {isRunningInContainer} from './container-detector' +import {getAuthConfigArgs} from './blacksmith-cache' + +/** + * gitfs serves the workspace from the git mirror sticky disk instead of + * checking it out: `gitfs mount --ref --daemon` + * returns as soon as the tree is mounted (milliseconds), writes the real + * checkout out in the background and swaps it in for the mount once it is + * complete. Writes made through the mount in the meantime (`.git`, npm's + * node_modules, ...) land in that same tree. The agent enables it per job via + * BLACKSMITH_USE_GITFS=true, driven by the installation's use_gitfs_checkout + * flag; anything gitfs cannot serve falls through to the regular checkout. + */ +export const GITFS_ENV = 'BLACKSMITH_USE_GITFS' +const GITFS_BINARY = 'gitfs' +const DAEMON_EXIT_TIMEOUT_MS = 10_000 +const DAEMON_KILL_TIMEOUT_MS = 5_000 + +export function isGitfsRequested( + env: {[key: string]: string | undefined} = process.env +): boolean { + return env[GITFS_ENV] === 'true' +} + +export interface EligibilitySettings { + ref: string + commit: string + submodules: boolean + lfs: boolean + sparseCheckout: string[] | null | undefined + filter: string | undefined + dissociate: boolean + repositoryPath: string +} + +export interface EligibilityContext { + mirrorPath: string | null + platform?: string + inContainer?: boolean +} + +/** + * Why gitfs cannot serve this checkout, or null when it can. The v1 cohort is + * the plain case: a Linux runner VM with the mirror mounted, a branch, tag, + * pull ref or commit to serve, and none of the features that need the + * workspace to be a real fetch (submodules, LFS, sparse/partial clones, + * dissociate). + */ +export function gitfsIneligibilityReason( + settings: EligibilitySettings, + ctx: EligibilityContext +): string | null { + const platform = ctx.platform ?? process.platform + if (platform !== 'linux') { + return `platform ${platform} is not linux` + } + if (!ctx.mirrorPath) { + return 'git mirror is not available' + } + const inContainer = ctx.inContainer ?? isRunningInContainer() + if (inContainer) { + return 'running inside a container' + } + if (!settings.ref && !settings.commit) { + return 'no ref or commit requested' + } + if (settings.submodules) { + return 'submodules are enabled' + } + if (settings.lfs) { + return 'lfs is enabled' + } + if (settings.sparseCheckout && settings.sparseCheckout.length > 0) { + return 'sparse checkout is enabled' + } + if (settings.filter) { + return `filter ${settings.filter} is set` + } + if (settings.dissociate) { + return 'dissociate is enabled' + } + if (fs.existsSync(path.join(settings.repositoryPath, '.git'))) { + return 'workspace already contains a repository' + } + return null +} + +export async function findBinary(): Promise { + const found = await io.which(GITFS_BINARY, false) + return found || null +} + +export type RefKind = 'branch' | 'tag' | 'pull' | 'other' | 'commit' + +export interface MirrorRevision { + sha: string + /** The mirror ref the revision came from, '' when only a commit matched. */ + mirrorRef: string + kind: RefKind +} + +interface Candidate { + rev: string + mirrorRef: string + kind: RefKind +} + +function classify(ref: string): RefKind { + const upper = ref.toUpperCase() + if (upper.startsWith('REFS/HEADS/')) return 'branch' + if (upper.startsWith('REFS/TAGS/')) return 'tag' + if (upper.startsWith('REFS/PULL/')) return 'pull' + return 'other' +} + +/** + * Revisions to try in the mirror for the requested ref/commit, in order. A + * commit wins over the ref (that is what the checkout would end up at); an + * unqualified ref is a branch first, then a tag, as in getCheckoutInfo. + */ +export function mirrorRevisionCandidates( + ref: string, + commit: string +): Candidate[] { + if (commit) { + const kind = ref ? classify(ref) : 'commit' + const mirrorRef = ref && ref.toUpperCase().startsWith('REFS/') ? ref : '' + return [{rev: `${commit}^{commit}`, mirrorRef, kind}] + } + if (!ref) { + return [] + } + if (ref.toUpperCase().startsWith('REFS/')) { + return [{rev: `${ref}^{commit}`, mirrorRef: ref, kind: classify(ref)}] + } + return [ + { + rev: `refs/heads/${ref}^{commit}`, + mirrorRef: `refs/heads/${ref}`, + kind: 'branch' + }, + { + rev: `refs/tags/${ref}^{commit}`, + mirrorRef: `refs/tags/${ref}`, + kind: 'tag' + } + ] +} + +/** + * Refspecs that bring the requested ref/commit into the mirror when it is + * not there yet (a push the mirror has not synced, or a pull-request merge + * ref). Qualified refs are fetched under their own name so the workspace + * can mirror them; a bare commit is fetched by sha. + */ +export function mirrorFetchRefspecs(ref: string, commit: string): string[] { + const qualified = !!ref && ref.toUpperCase().startsWith('REFS/') + if (commit) { + return qualified ? [`+${commit}:${ref}`] : [commit] + } + if (qualified) { + return [`+${ref}:${ref}`] + } + return [ + `+refs/heads/${ref}*:refs/heads/${ref}*`, + `+refs/tags/${ref}*:refs/tags/${ref}*` + ] +} + +export async function resolveInMirror( + mirrorPath: string, + candidates: Candidate[] +): Promise { + for (const candidate of candidates) { + const result = await exec.getExecOutput( + 'git', + ['-C', mirrorPath, 'rev-parse', '--verify', '--quiet', candidate.rev], + {silent: true, ignoreReturnCode: true} + ) + const sha = result.stdout.trim() + if (result.exitCode === 0 && /^[0-9a-f]{40}$/.test(sha)) { + return {sha, mirrorRef: candidate.mirrorRef, kind: candidate.kind} + } + } + return null +} + +export async function fetchIntoMirror( + mirrorPath: string, + repositoryUrl: string, + authToken: string, + refspecs: string[] +): Promise { + const auth = getAuthConfigArgs(repositoryUrl, authToken) + await exec.exec('git', [ + '-C', + mirrorPath, + '-c', + `${auth.configKey}=${auth.configValue}`, + '-c', + 'gc.auto=0', + 'fetch', + '--no-tags', + '--no-recurse-submodules', + '--no-write-fetch-head', + repositoryUrl, + ...refspecs + ]) +} + +export async function mount( + binary: string, + mirrorPath: string, + sha: string, + mountpoint: string +): Promise { + if (fs.existsSync(mountpoint) && fs.readdirSync(mountpoint).length > 0) { + throw new Error(`${mountpoint} is not empty`) + } + await exec.exec(binary, [ + 'mount', + mirrorPath, + '--ref', + sha, + mountpoint, + '--daemon' + ]) +} + +/** + * Point the workspace at the mounted revision without touching the tree: + * the local branch / tag / pull ref the regular checkout would have created, + * HEAD on it (or detached, as `git checkout` would leave it), and an index + * for the tree. The files are already there, served by gitfs. + */ +export async function pointWorkspaceAt( + repositoryPath: string, + ref: string, + revision: MirrorRevision, + env?: {[key: string]: string} +): Promise { + const git = async (...args: string[]): Promise => { + await exec.exec('git', ['-C', repositoryPath, ...args], {env}) + } + const {sha} = revision + if (revision.kind === 'branch') { + const branch = revision.mirrorRef.slice('refs/heads/'.length) + await git('update-ref', `refs/remotes/origin/${branch}`, sha) + await git('update-ref', `refs/heads/${branch}`, sha) + await git('config', `branch.${branch}.remote`, 'origin') + await git('config', `branch.${branch}.merge`, `refs/heads/${branch}`) + await git('symbolic-ref', 'HEAD', `refs/heads/${branch}`) + } else { + if (revision.kind === 'tag') { + await git('update-ref', revision.mirrorRef, sha) + } else if (revision.kind === 'pull') { + const name = revision.mirrorRef.slice('refs/pull/'.length) + await git('update-ref', `refs/remotes/pull/${name}`, sha) + } + await git('update-ref', '--no-deref', 'HEAD', sha) + } + // Index entries without stat data: git re-verifies file contents on the + // first status/diff instead of touching every file now. + await git('read-tree', sha) +} + +function daemonPids(mirrorPath: string): number[] { + const pids: number[] = [] + let entries: string[] + try { + entries = fs.readdirSync('/proc') + } catch { + return pids + } + for (const entry of entries) { + if (!/^\d+$/.test(entry)) continue + let cmdline: string + try { + cmdline = fs.readFileSync(`/proc/${entry}/cmdline`, 'utf8') + } catch { + continue + } + const argv = cmdline.split('\0') + if ( + path.basename(argv[0] || '') === GITFS_BINARY && + argv.includes('mount') && + argv.includes(mirrorPath) + ) { + pids.push(Number(entry)) + } + } + return pids +} + +async function waitForDaemons( + mirrorPath: string, + timeoutMs: number +): Promise { + const deadline = Date.now() + timeoutMs + let pids = daemonPids(mirrorPath) + while (pids.length > 0 && Date.now() < deadline) { + await new Promise(resolve => setTimeout(resolve, 100)) + pids = daemonPids(mirrorPath) + } + return pids +} + +/** + * Take gitfs down before the mirror sticky disk is unmounted: the daemon has + * the mirror's packs mapped, so the disk cannot be released while it lives. + * While the tree is still being written the workspace path is a symlink into + * the mount; unmounting then discards the partial tree and leaves the + * workspace empty. After the swap the path is a plain directory and the + * daemon exits on its own once nothing holds the old mount open. + */ +export async function teardown( + mountpoint: string, + mirrorPath: string +): Promise { + let isSymlink = false + try { + isSymlink = fs.lstatSync(mountpoint).isSymbolicLink() + } catch { + // mountpoint gone + } + if (isSymlink) { + const binary = await findBinary() + if (binary) { + core.info(`[gitfs] Unmounting ${mountpoint}`) + await exec.exec(binary, ['unmount', mountpoint], {ignoreReturnCode: true}) + } + } + let pids = await waitForDaemons(mirrorPath, DAEMON_EXIT_TIMEOUT_MS) + if (pids.length > 0) { + core.info( + `[gitfs] Stopping daemon(s) ${pids.join(', ')} still serving ${mirrorPath}` + ) + for (const pid of pids) { + try { + process.kill(pid, 'SIGTERM') + } catch { + // already gone + } + } + pids = await waitForDaemons(mirrorPath, DAEMON_KILL_TIMEOUT_MS) + if (pids.length > 0) { + core.warning( + `[gitfs] daemon(s) ${pids.join(', ')} did not exit; the mirror unmount may fail` + ) + } + } +} diff --git a/src/main.ts b/src/main.ts index f875dc9..80c1f6b 100644 --- a/src/main.ts +++ b/src/main.ts @@ -5,6 +5,7 @@ import * as inputHelper from './input-helper' import * as path from 'path' import * as stateHelper from './state-helper' import * as blacksmithCache from './blacksmith-cache' +import * as gitfs from './gitfs' import {checkPreviousStepFailures} from './step-checker' import {reportInternalMetric} from './internal-metrics' import * as mirrorTelemetry from './mirror-telemetry' @@ -46,6 +47,16 @@ async function cleanup(): Promise { const repoName = stateHelper.BlacksmithCacheRepoName const mountPoint = stateHelper.BlacksmithCacheMountPoint const mirrorPath = stateHelper.BlacksmithCacheMirrorPath + + // gitfs maps the mirror's packs, so it has to be gone before the sticky + // disk can be unmounted below. + if (stateHelper.GitfsMountPoint && mirrorPath) { + try { + await gitfs.teardown(stateHelper.GitfsMountPoint, mirrorPath) + } catch (error) { + core.warning(`[gitfs] teardown failed: ${error}`) + } + } const performedHydration = stateHelper.BlacksmithCachePerformedHydration let mirrorChanged = stateHelper.BlacksmithCacheMirrorChanged let mirrorSyncFailed = stateHelper.BlacksmithCacheMirrorSyncFailed diff --git a/src/state-helper.ts b/src/state-helper.ts index e420115..f1158f3 100644 --- a/src/state-helper.ts +++ b/src/state-helper.ts @@ -102,6 +102,12 @@ export const BlacksmithCacheVerbose = * Reason the host gave at mount time for denying this job's sticky disk commit * (e.g. branch protection). Empty when no denial was reported. */ +/** + * Workspace path served by gitfs, so the POST action can take the mount down + * before the mirror sticky disk is unmounted. + */ +export const GitfsMountPoint = core.getState('gitfsMountPoint') + export const BlacksmithCacheCommitEarlyDenyReason = core.getState( 'blacksmithCacheCommitEarlyDenyReason' ) @@ -233,6 +239,10 @@ export function setBlacksmithCacheCommitEarlyDenyReason(reason: string) { core.saveState('blacksmithCacheCommitEarlyDenyReason', reason) } +export function setGitfsMountPoint(mountPoint: string): void { + core.saveState('gitfsMountPoint', mountPoint) +} + // Publish a variable so that when the POST action runs, it can determine it should run the cleanup logic. // This is necessary since we don't have a separate entry point. if (!IsPost) { From e94ae5b28c15903e2c5659d830f80914bdd33e22 Mon Sep 17 00:00:00 2001 From: max Date: Sat, 17 Oct 2026 04:03:23 +0000 Subject: [PATCH 2/3] gitfs teardown: always run gitfs unmount (handles the post-swap hidden mount) Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- BLACKSMITH.md | 6 ++++-- dist/index.js | 28 ++++++++++------------------ src/gitfs.ts | 25 +++++++++---------------- 3 files changed, 23 insertions(+), 36 deletions(-) diff --git a/BLACKSMITH.md b/BLACKSMITH.md index 4c5af00..e198706 100644 --- a/BLACKSMITH.md +++ b/BLACKSMITH.md @@ -330,8 +330,10 @@ checking it out: produced) plus `git read-tree ` for the index. Git is usable from the first step; the first `git status` verifies file contents instead of trusting stat data. -4. The post step takes gitfs down before the mirror sticky disk is unmounted. - If the job ends before the swap, the partial tree is discarded. +4. The post step takes gitfs down before the mirror sticky disk is unmounted + (after the swap the FUSE view stays mounted on a hidden sibling path so + processes that were inside it keep a working cwd; `gitfs unmount` removes + it). If the job ends before the swap, the partial tree is discarded. gitfs is only used for the plain case — Linux runner VM (not inside a container), the mirror mounted, and no submodules, LFS, sparse checkout, diff --git a/dist/index.js b/dist/index.js index 87297cf..b2bd485 100644 --- a/dist/index.js +++ b/dist/index.js @@ -4574,26 +4574,18 @@ function waitForDaemons(mirrorPath, timeoutMs) { /** * Take gitfs down before the mirror sticky disk is unmounted: the daemon has * the mirror's packs mapped, so the disk cannot be released while it lives. - * While the tree is still being written the workspace path is a symlink into - * the mount; unmounting then discards the partial tree and leaves the - * workspace empty. After the swap the path is a plain directory and the - * daemon exits on its own once nothing holds the old mount open. + * `gitfs unmount` handles both phases — while the tree is still being + * written the workspace path is a symlink into the mount and unmounting + * discards the partial tree (the workspace is left empty); after the swap + * the path is a plain directory and the FUSE view that served it is still up + * on the hidden sibling path, which unmount takes down. */ function teardown(mountpoint, mirrorPath) { return __awaiter(this, void 0, void 0, function* () { - let isSymlink = false; - try { - isSymlink = fs.lstatSync(mountpoint).isSymbolicLink(); - } - catch (_a) { - // mountpoint gone - } - if (isSymlink) { - const binary = yield findBinary(); - if (binary) { - core.info(`[gitfs] Unmounting ${mountpoint}`); - yield exec.exec(binary, ['unmount', mountpoint], { ignoreReturnCode: true }); - } + const binary = yield findBinary(); + if (binary && fs.existsSync(mountpoint)) { + core.info(`[gitfs] Unmounting ${mountpoint}`); + yield exec.exec(binary, ['unmount', mountpoint], { ignoreReturnCode: true }); } let pids = yield waitForDaemons(mirrorPath, DAEMON_EXIT_TIMEOUT_MS); if (pids.length > 0) { @@ -4602,7 +4594,7 @@ function teardown(mountpoint, mirrorPath) { try { process.kill(pid, 'SIGTERM'); } - catch (_b) { + catch (_a) { // already gone } } diff --git a/src/gitfs.ts b/src/gitfs.ts index b8245ce..6f19c9f 100644 --- a/src/gitfs.ts +++ b/src/gitfs.ts @@ -313,27 +313,20 @@ async function waitForDaemons( /** * Take gitfs down before the mirror sticky disk is unmounted: the daemon has * the mirror's packs mapped, so the disk cannot be released while it lives. - * While the tree is still being written the workspace path is a symlink into - * the mount; unmounting then discards the partial tree and leaves the - * workspace empty. After the swap the path is a plain directory and the - * daemon exits on its own once nothing holds the old mount open. + * `gitfs unmount` handles both phases — while the tree is still being + * written the workspace path is a symlink into the mount and unmounting + * discards the partial tree (the workspace is left empty); after the swap + * the path is a plain directory and the FUSE view that served it is still up + * on the hidden sibling path, which unmount takes down. */ export async function teardown( mountpoint: string, mirrorPath: string ): Promise { - let isSymlink = false - try { - isSymlink = fs.lstatSync(mountpoint).isSymbolicLink() - } catch { - // mountpoint gone - } - if (isSymlink) { - const binary = await findBinary() - if (binary) { - core.info(`[gitfs] Unmounting ${mountpoint}`) - await exec.exec(binary, ['unmount', mountpoint], {ignoreReturnCode: true}) - } + const binary = await findBinary() + if (binary && fs.existsSync(mountpoint)) { + core.info(`[gitfs] Unmounting ${mountpoint}`) + await exec.exec(binary, ['unmount', mountpoint], {ignoreReturnCode: true}) } let pids = await waitForDaemons(mirrorPath, DAEMON_EXIT_TIMEOUT_MS) if (pids.length > 0) { From fbc7a1342cc9eaa82a14412b16450e1bf6bd6ac3 Mon Sep 17 00:00:00 2001 From: max Date: Mon, 19 Oct 2026 17:26:41 +0000 Subject: [PATCH 3/3] gitfs: stream a revision the mirror lacks instead of cloning or fetching first An empty sticky disk no longer costs a blocking git clone --mirror when the job uses gitfs: the action creates an empty bare mirror (same config as --mirror) and runs gitfs mount --ref --url , which fetches the revision --depth 1 into the mirror, serves the pack as it arrives and completes the mirror in the background. The same path replaces the blocking targeted fetch for a ref the mirror has not synced (pull-request merge refs). gitfs prints the mounted commit, which is how the action learns the sha before git has written the ref. The e2e test (self-skips without the binary) mounts from an empty mirror over file:// and checks the mirror ends up complete. Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- BLACKSMITH.md | 12 +++ __test__/gitfs.test.ts | 204 +++++++++++-------------------------- dist/index.js | 147 +++++++++++++++++++------- src/blacksmith-cache.ts | 29 ++++++ src/git-source-provider.ts | 90 +++++++++------- src/gitfs.ts | 67 +++++++++--- 6 files changed, 323 insertions(+), 226 deletions(-) diff --git a/BLACKSMITH.md b/BLACKSMITH.md index e198706..7917057 100644 --- a/BLACKSMITH.md +++ b/BLACKSMITH.md @@ -335,6 +335,18 @@ checking it out: processes that were inside it keep a working cwd; `gitfs unmount` removes it). If the job ends before the swap, the partial tree is discarded. +When the mirror lacks the revision — an empty sticky disk on the repository's +first job, a push the mirror has not synced, a pull-request merge ref — the +action does not hydrate with a blocking `git clone --mirror`. It creates an +empty bare mirror (first job) and runs `gitfs mount --ref +--url `: gitfs fetches the revision with `--depth 1` *into the mirror* +and serves the pack as it arrives, so the step returns as soon as the commit +is in (about a second for a 28 000-file repository from GitHub); when that +pack is complete gitfs fetches the rest of the repository in the background, +and the sticky disk is committed with a complete, ordinary mirror. The post +step's `gitfs unmount` ends a fetch still in flight; a mirror left shallow +that way is completed by the next job's mount. + gitfs is only used for the plain case — Linux runner VM (not inside a container), the mirror mounted, and no submodules, LFS, sparse checkout, `filter` or `dissociate` — and only when the `gitfs` binary is on the PATH. diff --git a/__test__/gitfs.test.ts b/__test__/gitfs.test.ts index dcc78ba..2c87cd4 100644 --- a/__test__/gitfs.test.ts +++ b/__test__/gitfs.test.ts @@ -138,153 +138,69 @@ describe('mirrorRevisionCandidates', () => { }) }) -describe('mirrorFetchRefspecs', () => { - const sha = 'c'.repeat(40) - it('fetches qualified refs under their own name and bare commits by sha', () => { - expect(gitfs.mirrorFetchRefspecs('refs/pull/7/merge', sha)).toEqual([ - `+${sha}:refs/pull/7/merge` - ]) - expect(gitfs.mirrorFetchRefspecs('refs/heads/dev', '')).toEqual([ - '+refs/heads/dev:refs/heads/dev' - ]) - expect(gitfs.mirrorFetchRefspecs('', sha)).toEqual([sha]) - expect(gitfs.mirrorFetchRefspecs('dev', sha)).toEqual([sha]) - expect(gitfs.mirrorFetchRefspecs('dev', '')).toEqual([ - '+refs/heads/dev*:refs/heads/dev*', - '+refs/tags/dev*:refs/tags/dev*' - ]) +describe('coldRevision', () => { + const sha = 'a'.repeat(40) + it('fetches a commit by sha, a qualified ref by name, a bare ref as a branch', () => { + expect(gitfs.coldRevision('refs/pull/7/merge', sha)).toBe(sha) + expect(gitfs.coldRevision('refs/pull/7/merge', '')).toBe( + 'refs/pull/7/merge' + ) + expect(gitfs.coldRevision('refs/tags/v1', '')).toBe('refs/tags/v1') + expect(gitfs.coldRevision('dev', '')).toBe('refs/heads/dev') }) }) -// End-to-end against a real mirror and the gitfs binary (skipped when gitfs -// is not on PATH): mount, give the workspace its .git through the mount, and -// check git is usable before and after the swap to the native tree. -const hasGitfs = ((): boolean => { - try { - execFileSync('gitfs', ['--help'], {stdio: 'ignore'}) - return true - } catch { - return false - } -})() - -const describeE2E = - hasGitfs && process.platform === 'linux' ? describe : describe.skip - -describeE2E('gitfs mount of a --mirror clone', () => { - jest.setTimeout(120_000) - let root: string - let mirror: string - let workspace: string - const git = (cwd: string, args: string[]): string => - execFileSync('git', ['-C', cwd, ...args], { - encoding: 'utf8', - env: { - ...process.env, - GIT_AUTHOR_NAME: 't', - GIT_AUTHOR_EMAIL: 't@t', - GIT_COMMITTER_NAME: 't', - GIT_COMMITTER_EMAIL: 't@t' - } - }).trim() - - beforeAll(() => { - root = fs.mkdtempSync(path.join(os.tmpdir(), 'gitfs-e2e-')) - const upstream = path.join(root, 'upstream') - fs.mkdirSync(upstream) - git(upstream, ['init', '-q', '-b', 'main']) - fs.mkdirSync(path.join(upstream, 'src')) - fs.writeFileSync(path.join(upstream, 'src', 'a.txt'), 'a\n') - fs.writeFileSync(path.join(upstream, 'README'), 'hi\n') - git(upstream, ['add', '.']) - git(upstream, ['commit', '-q', '-m', 'one']) - git(upstream, ['tag', 'v1']) - fs.writeFileSync(path.join(upstream, 'src', 'b.txt'), 'b\n') - git(upstream, ['add', '.']) - git(upstream, ['commit', '-q', '-m', 'two']) - mirror = path.join(root, 'owner-repo.git') - execFileSync('git', ['clone', '-q', '--mirror', upstream, mirror]) - workspace = path.join(root, 'work', 'repo') - fs.mkdirSync(workspace, {recursive: true}) - }) - - afterAll(async () => { - if (workspace) { - await gitfs.teardown(workspace, mirror) - } - fs.rmSync(root, {recursive: true, force: true}) - }) - - it('serves the branch tip with a working .git from the first command', async () => { +describe('mountCold (needs gitfs on PATH)', () => { + it('mounts a revision an empty mirror lacks and completes the mirror', async () => { const binary = await gitfs.findBinary() - expect(binary).not.toBeNull() - const revision = await gitfs.resolveInMirror( - mirror, - gitfs.mirrorRevisionCandidates('refs/heads/main', '') - ) - expect(revision).toEqual({ - sha: git(mirror, ['rev-parse', 'refs/heads/main']), - mirrorRef: 'refs/heads/main', - kind: 'branch' - }) - - await gitfs.mount(binary!, mirror, revision!.sha, workspace) - expect(fs.readFileSync(path.join(workspace, 'src', 'b.txt'), 'utf8')).toBe( - 'b\n' - ) - - git(workspace, ['init', '-q']) - fs.mkdirSync(path.join(workspace, '.git', 'objects', 'info'), { - recursive: true - }) - fs.writeFileSync( - path.join(workspace, '.git', 'objects', 'info', 'alternates'), - `${mirror}/objects\n` - ) - git(workspace, ['remote', 'add', 'origin', mirror]) - execFileSync('git', [ - '-C', - workspace, - 'fetch', - '-q', - '--no-tags', - mirror, - '+refs/heads/*:refs/remotes/origin/*', - '+refs/tags/*:refs/tags/*' - ]) - await gitfs.pointWorkspaceAt(workspace, 'refs/heads/main', revision!) - - expect(git(workspace, ['rev-parse', 'HEAD'])).toBe(revision!.sha) - expect(git(workspace, ['symbolic-ref', 'HEAD'])).toBe('refs/heads/main') - expect(git(workspace, ['rev-parse', 'refs/remotes/origin/main'])).toBe( - revision!.sha - ) - expect(git(workspace, ['status', '--porcelain'])).toBe('') - expect(git(workspace, ['log', '--oneline']).split('\n')).toHaveLength(2) - - // Wait for the swap: the workspace path becomes a plain directory. - const deadline = Date.now() + 60_000 - while (fs.lstatSync(workspace).isSymbolicLink() && Date.now() < deadline) { - await new Promise(r => setTimeout(r, 50)) + if (!binary) { + console.log('skipping: gitfs not on PATH') + return } - expect(fs.lstatSync(workspace).isDirectory()).toBe(true) - expect(git(workspace, ['status', '--porcelain'])).toBe('') - expect(git(workspace, ['rev-parse', 'HEAD'])).toBe(revision!.sha) - expect(fs.readFileSync(path.join(workspace, 'src', 'a.txt'), 'utf8')).toBe( - 'a\n' - ) - - // The mirror itself is untouched. - expect( - fs.existsSync(path.join(mirror, 'objects', 'info', 'alternates')) - ).toBe(false) - }) - - it('refuses a non-empty mountpoint', async () => { - const dir = fs.mkdtempSync(path.join(root, 'nonempty-')) - fs.writeFileSync(path.join(dir, 'x'), '') - await expect(gitfs.mount('gitfs', mirror, 'HEAD', dir)).rejects.toThrow( - /not empty/ - ) - }) + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'gitfs-cold-')) + const git = (cwd: string, ...args: string[]): string => + execFileSync('git', args, {cwd, stdio: 'pipe'}).toString().trim() + const author = ['-c', 'user.name=t', '-c', 'user.email=t@t'] + const work = path.join(root, 'work') + fs.mkdirSync(work) + git(work, 'init', '-q', '-b', 'main') + git(work, ...author, 'commit', '-q', '--allow-empty', '-m', 'first') + fs.writeFileSync(path.join(work, 'hello.txt'), 'hello\n') + git(work, 'add', '-A') + git(work, ...author, 'commit', '-q', '-m', 'second') + const upstream = path.join(root, 'upstream.git') + git(root, 'clone', '-q', '--bare', work, upstream) + const url = `file://${upstream}` + const mirror = path.join(root, 'mirror.git') + git(root, 'init', '-q', '--bare', mirror) + git(mirror, 'remote', 'add', '--mirror=fetch', 'origin', url) + const mnt = path.join(root, 'ws') + fs.mkdirSync(mnt) + try { + const sha = await gitfs.mountCold( + binary, + mirror, + 'refs/heads/main', + mnt, + url, + '' + ) + expect(sha).toBe(git(upstream, 'rev-parse', 'refs/heads/main')) + expect(fs.readFileSync(path.join(mnt, 'hello.txt'), 'utf8')).toBe( + 'hello\n' + ) + const deadline = Date.now() + 30000 + while ( + fs.existsSync(path.join(mirror, 'shallow')) && + Date.now() < deadline + ) { + await new Promise(resolve => setTimeout(resolve, 50)) + } + expect(fs.existsSync(path.join(mirror, 'shallow'))).toBe(false) + expect(git(mirror, 'rev-list', '--count', 'main')).toBe('2') + } finally { + await gitfs.teardown(mnt, mirror) + fs.rmSync(root, {recursive: true, force: true}) + } + }, 60000) }) diff --git a/dist/index.js b/dist/index.js index b2bd485..33f11c8 100644 --- a/dist/index.js +++ b/dist/index.js @@ -50,6 +50,7 @@ exports.getMirrorPath = getMirrorPath; exports.stickyDiskKeyFor = stickyDiskKeyFor; exports.setupCache = setupCache; exports.getAuthConfigArgs = getAuthConfigArgs; +exports.initEmptyMirror = initEmptyMirror; exports.ensureMirror = ensureMirror; exports.diffMirrorRefs = diffMirrorRefs; exports.purgePullRefs = purgePullRefs; @@ -501,6 +502,34 @@ function summarizeTrace2Perf(trace2PerfPath, title) { fs.rmSync(trace2PerfPath, { force: true }); } } +/** + * An empty bare mirror with `git clone --mirror`'s config, for gitfs to + * stream the first job's revision into; gitfs completes the mirror in the + * background, so it is a full mirror by the time the job ends. + */ +function initEmptyMirror(mirrorPath, repoUrl) { + return __awaiter(this, void 0, void 0, function* () { + var _a, _b, _c, _d; + const mirrorDir = path.dirname(mirrorPath); + yield exec.exec('sudo', ['mkdir', '-p', mirrorDir]); + const uid = (_b = (_a = process.getuid) === null || _a === void 0 ? void 0 : _a.call(process)) !== null && _b !== void 0 ? _b : 1000; + const gid = (_d = (_c = process.getgid) === null || _c === void 0 ? void 0 : _c.call(process)) !== null && _d !== void 0 ? _d : 1000; + yield exec.exec('sudo', ['chown', '-R', `${uid}:${gid}`, mirrorDir]); + if (fs.existsSync(mirrorPath)) { + yield fs.promises.rm(mirrorPath, { recursive: true, force: true }); + } + yield exec.exec('git', ['init', '-q', '--bare', mirrorPath]); + yield exec.exec('git', [ + '-C', + mirrorPath, + 'remote', + 'add', + '--mirror=fetch', + 'origin', + repoUrl + ]); + }); +} /** * Ensure a bare git mirror exists. If the mirror doesn't exist, clone it. * If the mirror already exists, it is left as-is; the caller brings it up to @@ -3708,6 +3737,8 @@ function getSourceInner(settings, report, onHydrationReport) { // of this run (hydrated now, or synced via ls-remote diff). Only then can // the workspace copy its refs from the mirror instead of the network. let mirrorFresh = false; + // gitfs is hydrating an empty mirror: it has no refs to populate from. + let gitfsHydrates = false; if (blacksmithCache.shouldUseBlacksmithCache()) { const setupStart = Date.now(); try { @@ -3733,7 +3764,22 @@ function getSourceInner(settings, report, onHydrationReport) { const cloneStart = Date.now(); let performedHydration = false; try { - performedHydration = yield blacksmithCache.ensureMirror(cacheInfo.mirrorPath, repositoryUrl, settings.authToken, settings.verbose); + if (!mirrorExisted && + gitfs.isGitfsRequested() && + !gitfs.gitfsIneligibilityReason(settings, { + mirrorPath: cacheInfo.mirrorPath + }) && + (yield gitfs.findBinary())) { + // No blocking clone: gitfs streams the job's revision into the + // empty mirror and completes it in the background. + core.info(`[gitfs] no mirror at ${cacheInfo.mirrorPath}: gitfs hydrates it`); + yield blacksmithCache.initEmptyMirror(cacheInfo.mirrorPath, repositoryUrl); + performedHydration = true; + gitfsHydrates = true; + } + else { + performedHydration = yield blacksmithCache.ensureMirror(cacheInfo.mirrorPath, repositoryUrl, settings.authToken, settings.verbose); + } } catch (error) { if (!mirrorExisted) { @@ -3768,7 +3814,7 @@ function getSourceInner(settings, report, onHydrationReport) { } if (performedHydration) { // A freshly-cloned mirror is exactly the remote's current state - mirrorFresh = true; + mirrorFresh = !gitfsHydrates; } else if (settings.fetchDepth <= 0) { // Bring the mirror's branch/tag refs up to date with the remote @@ -3952,27 +3998,27 @@ function mountWithGitfs(settings, cacheInfo, repositoryUrl) { return null; } const candidates = gitfs.mirrorRevisionCandidates(settings.ref, settings.commit); - let revision = yield gitfs.resolveInMirror(cacheInfo.mirrorPath, candidates); - if (!revision) { - // The mirror does not have the requested revision yet (a push it has - // not synced, a pull-request merge ref): fetch just that into the - // mirror, where the regular path would have fetched it into the - // workspace. The mirror changed, so the post step commits it. - const refspecs = gitfs.mirrorFetchRefspecs(settings.ref, settings.commit); - core.info(`[gitfs] ${settings.commit || settings.ref} is not in the mirror, fetching ${refspecs.join(' ')}`); - yield gitfs.fetchIntoMirror(cacheInfo.mirrorPath, repositoryUrl, settings.authToken, refspecs); - stateHelper.setBlacksmithCacheMirrorChanged(true); - revision = yield gitfs.resolveInMirror(cacheInfo.mirrorPath, candidates); - } - if (!revision) { - core.info(`[gitfs] not used: ${settings.commit || settings.ref} could not be resolved in the mirror`); - return null; - } + const revision = yield gitfs.resolveInMirror(cacheInfo.mirrorPath, candidates); const mountStart = Date.now(); - yield gitfs.mount(binary, cacheInfo.mirrorPath, revision.sha, settings.repositoryPath); + if (revision) { + yield gitfs.mount(binary, cacheInfo.mirrorPath, revision.sha, settings.repositoryPath); + stateHelper.setGitfsMountPoint(settings.repositoryPath); + core.info(`[gitfs] ${settings.repositoryPath} serves ${revision.sha} from the mirror (mounted in ${Date.now() - mountStart} ms); the tree is written out in the background`); + return revision; + } + // The mirror lacks the revision (an empty mirror, a push it has not + // synced, a pull-request merge ref): gitfs fetches it from the remote into + // the mirror and serves the pack as it arrives; the rest of the repository + // follows in the background. The mirror changes, so the post step commits + // it. + const rev = gitfs.coldRevision(settings.ref, settings.commit); + core.info(`[gitfs] ${rev} is not in the mirror, streaming it from the remote`); + const sha = yield gitfs.mountCold(binary, cacheInfo.mirrorPath, rev, settings.repositoryPath, repositoryUrl, settings.authToken); stateHelper.setGitfsMountPoint(settings.repositoryPath); - core.info(`[gitfs] ${settings.repositoryPath} serves ${revision.sha} from the mirror (mounted in ${Date.now() - mountStart} ms); the tree is written out in the background`); - return revision; + stateHelper.setBlacksmithCacheMirrorChanged(true); + const { mirrorRef, kind } = candidates[0]; + core.info(`[gitfs] ${settings.repositoryPath} serves ${sha} streamed from the remote (mounted in ${Date.now() - mountStart} ms); the mirror is completed in the background`); + return { sha, mirrorRef, kind }; }); } /** @@ -4313,10 +4359,11 @@ exports.isGitfsRequested = isGitfsRequested; exports.gitfsIneligibilityReason = gitfsIneligibilityReason; exports.findBinary = findBinary; exports.mirrorRevisionCandidates = mirrorRevisionCandidates; -exports.mirrorFetchRefspecs = mirrorFetchRefspecs; +exports.coldRevision = coldRevision; exports.resolveInMirror = resolveInMirror; exports.fetchIntoMirror = fetchIntoMirror; exports.mount = mount; +exports.mountCold = mountCold; exports.pointWorkspaceAt = pointWorkspaceAt; exports.teardown = teardown; const core = __importStar(__nccwpck_require__(2186)); @@ -4433,23 +4480,17 @@ function mirrorRevisionCandidates(ref, commit) { ]; } /** - * Refspecs that bring the requested ref/commit into the mirror when it is - * not there yet (a push the mirror has not synced, or a pull-request merge - * ref). Qualified refs are fetched under their own name so the workspace - * can mirror them; a bare commit is fetched by sha. + * What `gitfs mount --url` fetches for a revision the mirror lacks: a commit + * by sha, a qualified ref by name, an unqualified ref as a branch. */ -function mirrorFetchRefspecs(ref, commit) { - const qualified = !!ref && ref.toUpperCase().startsWith('REFS/'); +function coldRevision(ref, commit) { if (commit) { - return qualified ? [`+${commit}:${ref}`] : [commit]; + return commit; } - if (qualified) { - return [`+${ref}:${ref}`]; + if (ref.toUpperCase().startsWith('REFS/')) { + return ref; } - return [ - `+refs/heads/${ref}*:refs/heads/${ref}*`, - `+refs/tags/${ref}*:refs/tags/${ref}*` - ]; + return `refs/heads/${ref}`; } function resolveInMirror(mirrorPath, candidates) { return __awaiter(this, void 0, void 0, function* () { @@ -4497,6 +4538,42 @@ function mount(binary, mirrorPath, sha, mountpoint) { ]); }); } +/** + * Mount a revision the mirror does not have: gitfs fetches it from the + * remote into the mirror (`--depth 1`, serving the pack as it arrives) and + * completes the mirror in the background. Returns the commit sha, which + * gitfs prints once the commit is in, before git has written the ref. + */ +function mountCold(binary, mirrorPath, rev, mountpoint, repositoryUrl, authToken) { + return __awaiter(this, void 0, void 0, function* () { + var _a, _b; + if (fs.existsSync(mountpoint) && fs.readdirSync(mountpoint).length > 0) { + throw new Error(`${mountpoint} is not empty`); + } + const env = {}; + for (const [key, value] of Object.entries(process.env)) { + if (value !== undefined) { + env[key] = value; + } + } + env.GITFS_TOKEN = authToken; + const result = yield exec.getExecOutput(binary, [ + 'mount', + mirrorPath, + '--ref', + rev, + mountpoint, + '--daemon', + '--url', + repositoryUrl + ], { env }); + const sha = (_b = (_a = result.stdout.trim().split('\n').pop()) === null || _a === void 0 ? void 0 : _a.trim()) !== null && _b !== void 0 ? _b : ''; + if (!/^[0-9a-f]{40}$/.test(sha)) { + throw new Error(`gitfs did not report the mounted commit: ${result.stdout.trim()}`); + } + return sha; + }); +} /** * Point the workspace at the mounted revision without touching the tree: * the local branch / tag / pull ref the regular checkout would have created, diff --git a/src/blacksmith-cache.ts b/src/blacksmith-cache.ts index bef72d6..d290f70 100644 --- a/src/blacksmith-cache.ts +++ b/src/blacksmith-cache.ts @@ -548,6 +548,35 @@ function summarizeTrace2Perf(trace2PerfPath: string, title: string): void { } } +/** + * An empty bare mirror with `git clone --mirror`'s config, for gitfs to + * stream the first job's revision into; gitfs completes the mirror in the + * background, so it is a full mirror by the time the job ends. + */ +export async function initEmptyMirror( + mirrorPath: string, + repoUrl: string +): Promise { + const mirrorDir = path.dirname(mirrorPath) + await exec.exec('sudo', ['mkdir', '-p', mirrorDir]) + const uid = process.getuid?.() ?? 1000 + const gid = process.getgid?.() ?? 1000 + await exec.exec('sudo', ['chown', '-R', `${uid}:${gid}`, mirrorDir]) + if (fs.existsSync(mirrorPath)) { + await fs.promises.rm(mirrorPath, {recursive: true, force: true}) + } + await exec.exec('git', ['init', '-q', '--bare', mirrorPath]) + await exec.exec('git', [ + '-C', + mirrorPath, + 'remote', + 'add', + '--mirror=fetch', + 'origin', + repoUrl + ]) +} + /** * Ensure a bare git mirror exists. If the mirror doesn't exist, clone it. * If the mirror already exists, it is left as-is; the caller brings it up to diff --git a/src/git-source-provider.ts b/src/git-source-provider.ts index 83de3bd..0d5faff 100644 --- a/src/git-source-provider.ts +++ b/src/git-source-provider.ts @@ -161,6 +161,8 @@ async function getSourceInner( // of this run (hydrated now, or synced via ls-remote diff). Only then can // the workspace copy its refs from the mirror instead of the network. let mirrorFresh = false + // gitfs is hydrating an empty mirror: it has no refs to populate from. + let gitfsHydrates = false if (blacksmithCache.shouldUseBlacksmithCache()) { const setupStart = Date.now() try { @@ -194,12 +196,33 @@ async function getSourceInner( const cloneStart = Date.now() let performedHydration = false try { - performedHydration = await blacksmithCache.ensureMirror( - cacheInfo.mirrorPath, - repositoryUrl, - settings.authToken, - settings.verbose - ) + if ( + !mirrorExisted && + gitfs.isGitfsRequested() && + !gitfs.gitfsIneligibilityReason(settings, { + mirrorPath: cacheInfo.mirrorPath + }) && + (await gitfs.findBinary()) + ) { + // No blocking clone: gitfs streams the job's revision into the + // empty mirror and completes it in the background. + core.info( + `[gitfs] no mirror at ${cacheInfo.mirrorPath}: gitfs hydrates it` + ) + await blacksmithCache.initEmptyMirror( + cacheInfo.mirrorPath, + repositoryUrl + ) + performedHydration = true + gitfsHydrates = true + } else { + performedHydration = await blacksmithCache.ensureMirror( + cacheInfo.mirrorPath, + repositoryUrl, + settings.authToken, + settings.verbose + ) + } } catch (error) { if (!mirrorExisted) { onHydrationReport({ @@ -235,7 +258,7 @@ async function getSourceInner( if (performedHydration) { // A freshly-cloned mirror is exactly the remote's current state - mirrorFresh = true + mirrorFresh = !gitfsHydrates } else if (settings.fetchDepth <= 0) { // Bring the mirror's branch/tag refs up to date with the remote // (ls-remote diff + targeted fetch of only the changed refs), so @@ -486,44 +509,43 @@ async function mountWithGitfs( settings.ref, settings.commit ) - let revision = await gitfs.resolveInMirror(cacheInfo.mirrorPath, candidates) - if (!revision) { - // The mirror does not have the requested revision yet (a push it has - // not synced, a pull-request merge ref): fetch just that into the - // mirror, where the regular path would have fetched it into the - // workspace. The mirror changed, so the post step commits it. - const refspecs = gitfs.mirrorFetchRefspecs(settings.ref, settings.commit) - core.info( - `[gitfs] ${settings.commit || settings.ref} is not in the mirror, fetching ${refspecs.join(' ')}` - ) - await gitfs.fetchIntoMirror( + const revision = await gitfs.resolveInMirror(cacheInfo.mirrorPath, candidates) + const mountStart = Date.now() + if (revision) { + await gitfs.mount( + binary, cacheInfo.mirrorPath, - repositoryUrl, - settings.authToken, - refspecs + revision.sha, + settings.repositoryPath ) - stateHelper.setBlacksmithCacheMirrorChanged(true) - revision = await gitfs.resolveInMirror(cacheInfo.mirrorPath, candidates) - } - if (!revision) { + stateHelper.setGitfsMountPoint(settings.repositoryPath) core.info( - `[gitfs] not used: ${settings.commit || settings.ref} could not be resolved in the mirror` + `[gitfs] ${settings.repositoryPath} serves ${revision.sha} from the mirror (mounted in ${Date.now() - mountStart} ms); the tree is written out in the background` ) - return null + return revision } - - const mountStart = Date.now() - await gitfs.mount( + // The mirror lacks the revision (an empty mirror, a push it has not + // synced, a pull-request merge ref): gitfs fetches it from the remote into + // the mirror and serves the pack as it arrives; the rest of the repository + // follows in the background. The mirror changes, so the post step commits + // it. + const rev = gitfs.coldRevision(settings.ref, settings.commit) + core.info(`[gitfs] ${rev} is not in the mirror, streaming it from the remote`) + const sha = await gitfs.mountCold( binary, cacheInfo.mirrorPath, - revision.sha, - settings.repositoryPath + rev, + settings.repositoryPath, + repositoryUrl, + settings.authToken ) stateHelper.setGitfsMountPoint(settings.repositoryPath) + stateHelper.setBlacksmithCacheMirrorChanged(true) + const {mirrorRef, kind} = candidates[0] core.info( - `[gitfs] ${settings.repositoryPath} serves ${revision.sha} from the mirror (mounted in ${Date.now() - mountStart} ms); the tree is written out in the background` + `[gitfs] ${settings.repositoryPath} serves ${sha} streamed from the remote (mounted in ${Date.now() - mountStart} ms); the mirror is completed in the background` ) - return revision + return {sha, mirrorRef, kind} } /** diff --git a/src/gitfs.ts b/src/gitfs.ts index 6f19c9f..656d26e 100644 --- a/src/gitfs.ts +++ b/src/gitfs.ts @@ -153,23 +153,17 @@ export function mirrorRevisionCandidates( } /** - * Refspecs that bring the requested ref/commit into the mirror when it is - * not there yet (a push the mirror has not synced, or a pull-request merge - * ref). Qualified refs are fetched under their own name so the workspace - * can mirror them; a bare commit is fetched by sha. + * What `gitfs mount --url` fetches for a revision the mirror lacks: a commit + * by sha, a qualified ref by name, an unqualified ref as a branch. */ -export function mirrorFetchRefspecs(ref: string, commit: string): string[] { - const qualified = !!ref && ref.toUpperCase().startsWith('REFS/') +export function coldRevision(ref: string, commit: string): string { if (commit) { - return qualified ? [`+${commit}:${ref}`] : [commit] + return commit } - if (qualified) { - return [`+${ref}:${ref}`] + if (ref.toUpperCase().startsWith('REFS/')) { + return ref } - return [ - `+refs/heads/${ref}*:refs/heads/${ref}*`, - `+refs/tags/${ref}*:refs/tags/${ref}*` - ] + return `refs/heads/${ref}` } export async function resolveInMirror( @@ -232,6 +226,53 @@ export async function mount( ]) } +/** + * Mount a revision the mirror does not have: gitfs fetches it from the + * remote into the mirror (`--depth 1`, serving the pack as it arrives) and + * completes the mirror in the background. Returns the commit sha, which + * gitfs prints once the commit is in, before git has written the ref. + */ +export async function mountCold( + binary: string, + mirrorPath: string, + rev: string, + mountpoint: string, + repositoryUrl: string, + authToken: string +): Promise { + if (fs.existsSync(mountpoint) && fs.readdirSync(mountpoint).length > 0) { + throw new Error(`${mountpoint} is not empty`) + } + const env: {[key: string]: string} = {} + for (const [key, value] of Object.entries(process.env)) { + if (value !== undefined) { + env[key] = value + } + } + env.GITFS_TOKEN = authToken + const result = await exec.getExecOutput( + binary, + [ + 'mount', + mirrorPath, + '--ref', + rev, + mountpoint, + '--daemon', + '--url', + repositoryUrl + ], + {env} + ) + const sha = result.stdout.trim().split('\n').pop()?.trim() ?? '' + if (!/^[0-9a-f]{40}$/.test(sha)) { + throw new Error( + `gitfs did not report the mounted commit: ${result.stdout.trim()}` + ) + } + return sha +} + /** * Point the workspace at the mounted revision without touching the tree: * the local branch / tag / pull ref the regular checkout would have created,