diff --git a/BLACKSMITH.md b/BLACKSMITH.md index b271978..7917057 100644 --- a/BLACKSMITH.md +++ b/BLACKSMITH.md @@ -307,6 +307,53 @@ The mirror stays in sync with GitHub through **incremental fetches**: - Redundant pack files (consolidated into single optimized pack) - All unreachable objects (removed immediately via `git gc --prune=now`) +## Serving the Workspace with gitfs + +When the job's installation has `use_gitfs_checkout` enabled, the agent +exports `BLACKSMITH_USE_GITFS=true` into the runner environment and the +action serves the workspace from the mirror with +[gitfs](https://github.com/useblacksmith/gitfs) instead of fetching and +checking it out: + +1. The requested ref/commit is resolved in the mirror. If the mirror does not + have it yet (a push it has not synced, a pull-request merge ref), only that + ref is fetched *into the mirror*, where the regular path would have fetched + it into the workspace. +2. `gitfs mount --ref $GITHUB_WORKSPACE --daemon` mounts the + tree in milliseconds. gitfs writes the real checkout out in the background + and atomically swaps it in for the mount once it is complete (seconds, even + for 100k-file trees); until then reads are served lazily from the mirror's + packs and writes go straight into the tree being written. +3. `.git` is created through the mount exactly as in the regular path + (`git init`, alternates to the mirror, `origin`, auth, the mirror's + branch/tag refs, the local branch or detached HEAD the checkout would have + produced) plus `git read-tree ` for the index. Git is usable from the + first step; the first `git status` verifies file contents instead of + trusting stat data. +4. The post step takes gitfs down before the mirror sticky disk is unmounted + (after the swap the FUSE view stays mounted on a hidden sibling path so + processes that were inside it keep a working cwd; `gitfs unmount` removes + it). If the job ends before the swap, the partial tree is discarded. + +When the mirror lacks the revision — an empty sticky disk on the repository's +first job, a push the mirror has not synced, a pull-request merge ref — the +action does not hydrate with a blocking `git clone --mirror`. It creates an +empty bare mirror (first job) and runs `gitfs mount --ref +--url `: gitfs fetches the revision with `--depth 1` *into the mirror* +and serves the pack as it arrives, so the step returns as soon as the commit +is in (about a second for a 28 000-file repository from GitHub); when that +pack is complete gitfs fetches the rest of the repository in the background, +and the sticky disk is committed with a complete, ordinary mirror. The post +step's `gitfs unmount` ends a fetch still in flight; a mirror left shallow +that way is completed by the next job's mount. + +gitfs is only used for the plain case — Linux runner VM (not inside a +container), the mirror mounted, and no submodules, LFS, sparse checkout, +`filter` or `dissociate` — and only when the `gitfs` binary is on the PATH. +Everything else, and any gitfs failure, falls through to the regular fetch + +checkout. Because objects come from the mirror through alternates, the +workspace has the full history regardless of `fetch-depth`. + ## The `dissociate` Option When to use `dissociate: true`: diff --git a/__test__/gitfs.test.ts b/__test__/gitfs.test.ts new file mode 100644 index 0000000..2c87cd4 --- /dev/null +++ b/__test__/gitfs.test.ts @@ -0,0 +1,206 @@ +// gitfs.ts pulls in blacksmith-cache for the auth header; stub its gRPC +// dependencies the same way blacksmith-cache.test.ts does. +jest.mock('@connectrpc/connect', () => ({ + createClient: jest.fn(), + ConnectError: class ConnectError extends Error {}, + Code: {Aborted: 'ABORTED'} +})) +jest.mock('@connectrpc/connect-node', () => ({ + createGrpcTransport: jest.fn() +})) +jest.mock( + '@buf/blacksmith_vm-agent.connectrpc_es/stickydisk/v1/stickydisk_connect', + () => ({StickyDiskService: {}}) +) + +import * as fs from 'fs' +import * as os from 'os' +import * as path from 'path' +import {execFileSync} from 'child_process' +import * as gitfs from '../src/gitfs' + +const settings = ( + overrides: Partial = {} +): gitfs.EligibilitySettings => ({ + ref: 'refs/heads/main', + commit: '', + submodules: false, + lfs: false, + sparseCheckout: [], + filter: undefined, + dissociate: false, + repositoryPath: path.join(os.tmpdir(), 'gitfs-test-does-not-exist'), + ...overrides +}) + +const ctx: gitfs.EligibilityContext = { + mirrorPath: '/mnt/git-mirror/owner-repo.git', + platform: 'linux', + inContainer: false +} + +describe('isGitfsRequested', () => { + it('is driven by BLACKSMITH_USE_GITFS=true only', () => { + expect(gitfs.isGitfsRequested({})).toBe(false) + expect(gitfs.isGitfsRequested({BLACKSMITH_USE_GITFS: 'false'})).toBe(false) + expect(gitfs.isGitfsRequested({BLACKSMITH_USE_GITFS: 'TRUE'})).toBe(false) + expect(gitfs.isGitfsRequested({BLACKSMITH_USE_GITFS: 'true'})).toBe(true) + }) +}) + +describe('gitfsIneligibilityReason', () => { + it('accepts the plain linux + mirror case', () => { + expect(gitfs.gitfsIneligibilityReason(settings(), ctx)).toBeNull() + expect( + gitfs.gitfsIneligibilityReason( + settings({ref: '', commit: 'a'.repeat(40)}), + ctx + ) + ).toBeNull() + }) + + it('needs a mounted mirror on a linux runner outside a container', () => { + expect( + gitfs.gitfsIneligibilityReason(settings(), {...ctx, mirrorPath: null}) + ).toMatch(/mirror/) + expect( + gitfs.gitfsIneligibilityReason(settings(), {...ctx, platform: 'darwin'}) + ).toMatch(/linux/) + expect( + gitfs.gitfsIneligibilityReason(settings(), {...ctx, inContainer: true}) + ).toMatch(/container/) + }) + + it('leaves features the fetch path implements to the fetch path', () => { + expect( + gitfs.gitfsIneligibilityReason(settings({submodules: true}), ctx) + ).toMatch(/submodules/) + expect(gitfs.gitfsIneligibilityReason(settings({lfs: true}), ctx)).toMatch( + /lfs/ + ) + expect( + gitfs.gitfsIneligibilityReason(settings({sparseCheckout: ['src']}), ctx) + ).toMatch(/sparse/) + expect( + gitfs.gitfsIneligibilityReason(settings({filter: 'blob:none'}), ctx) + ).toMatch(/filter/) + expect( + gitfs.gitfsIneligibilityReason(settings({dissociate: true}), ctx) + ).toMatch(/dissociate/) + expect( + gitfs.gitfsIneligibilityReason(settings({ref: '', commit: ''}), ctx) + ).toMatch(/ref or commit/) + }) + + it('does not mount over an existing repository', () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'gitfs-elig-')) + try { + fs.mkdirSync(path.join(dir, '.git')) + expect( + gitfs.gitfsIneligibilityReason(settings({repositoryPath: dir}), ctx) + ).toMatch(/already contains/) + } finally { + fs.rmSync(dir, {recursive: true, force: true}) + } + }) +}) + +describe('mirrorRevisionCandidates', () => { + const sha = 'b'.repeat(40) + + it('prefers the commit and keeps the ref it belongs to', () => { + expect(gitfs.mirrorRevisionCandidates('refs/heads/main', sha)).toEqual([ + {rev: `${sha}^{commit}`, mirrorRef: 'refs/heads/main', kind: 'branch'} + ]) + expect(gitfs.mirrorRevisionCandidates('refs/pull/42/merge', sha)).toEqual([ + {rev: `${sha}^{commit}`, mirrorRef: 'refs/pull/42/merge', kind: 'pull'} + ]) + expect(gitfs.mirrorRevisionCandidates('', sha)).toEqual([ + {rev: `${sha}^{commit}`, mirrorRef: '', kind: 'commit'} + ]) + // An unqualified ref alongside a commit does not name a mirror ref. + expect(gitfs.mirrorRevisionCandidates('main', sha)).toEqual([ + {rev: `${sha}^{commit}`, mirrorRef: '', kind: 'other'} + ]) + }) + + it('peels qualified refs to commits', () => { + expect(gitfs.mirrorRevisionCandidates('refs/tags/v1', '')).toEqual([ + {rev: 'refs/tags/v1^{commit}', mirrorRef: 'refs/tags/v1', kind: 'tag'} + ]) + }) + + it('tries an unqualified ref as a branch, then a tag', () => { + expect( + gitfs.mirrorRevisionCandidates('v1', '').map(c => c.mirrorRef) + ).toEqual(['refs/heads/v1', 'refs/tags/v1']) + expect(gitfs.mirrorRevisionCandidates('', '')).toEqual([]) + }) +}) + +describe('coldRevision', () => { + const sha = 'a'.repeat(40) + it('fetches a commit by sha, a qualified ref by name, a bare ref as a branch', () => { + expect(gitfs.coldRevision('refs/pull/7/merge', sha)).toBe(sha) + expect(gitfs.coldRevision('refs/pull/7/merge', '')).toBe( + 'refs/pull/7/merge' + ) + expect(gitfs.coldRevision('refs/tags/v1', '')).toBe('refs/tags/v1') + expect(gitfs.coldRevision('dev', '')).toBe('refs/heads/dev') + }) +}) + +describe('mountCold (needs gitfs on PATH)', () => { + it('mounts a revision an empty mirror lacks and completes the mirror', async () => { + const binary = await gitfs.findBinary() + if (!binary) { + console.log('skipping: gitfs not on PATH') + return + } + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'gitfs-cold-')) + const git = (cwd: string, ...args: string[]): string => + execFileSync('git', args, {cwd, stdio: 'pipe'}).toString().trim() + const author = ['-c', 'user.name=t', '-c', 'user.email=t@t'] + const work = path.join(root, 'work') + fs.mkdirSync(work) + git(work, 'init', '-q', '-b', 'main') + git(work, ...author, 'commit', '-q', '--allow-empty', '-m', 'first') + fs.writeFileSync(path.join(work, 'hello.txt'), 'hello\n') + git(work, 'add', '-A') + git(work, ...author, 'commit', '-q', '-m', 'second') + const upstream = path.join(root, 'upstream.git') + git(root, 'clone', '-q', '--bare', work, upstream) + const url = `file://${upstream}` + const mirror = path.join(root, 'mirror.git') + git(root, 'init', '-q', '--bare', mirror) + git(mirror, 'remote', 'add', '--mirror=fetch', 'origin', url) + const mnt = path.join(root, 'ws') + fs.mkdirSync(mnt) + try { + const sha = await gitfs.mountCold( + binary, + mirror, + 'refs/heads/main', + mnt, + url, + '' + ) + expect(sha).toBe(git(upstream, 'rev-parse', 'refs/heads/main')) + expect(fs.readFileSync(path.join(mnt, 'hello.txt'), 'utf8')).toBe( + 'hello\n' + ) + const deadline = Date.now() + 30000 + while ( + fs.existsSync(path.join(mirror, 'shallow')) && + Date.now() < deadline + ) { + await new Promise(resolve => setTimeout(resolve, 50)) + } + expect(fs.existsSync(path.join(mirror, 'shallow'))).toBe(false) + expect(git(mirror, 'rev-list', '--count', 'main')).toBe('2') + } finally { + await gitfs.teardown(mnt, mirror) + fs.rmSync(root, {recursive: true, force: true}) + } + }, 60000) +}) diff --git a/dist/index.js b/dist/index.js index d1d1519..33f11c8 100644 --- a/dist/index.js +++ b/dist/index.js @@ -49,6 +49,8 @@ exports.shouldUseBlacksmithCache = shouldUseBlacksmithCache; exports.getMirrorPath = getMirrorPath; exports.stickyDiskKeyFor = stickyDiskKeyFor; exports.setupCache = setupCache; +exports.getAuthConfigArgs = getAuthConfigArgs; +exports.initEmptyMirror = initEmptyMirror; exports.ensureMirror = ensureMirror; exports.diffMirrorRefs = diffMirrorRefs; exports.purgePullRefs = purgePullRefs; @@ -500,6 +502,34 @@ function summarizeTrace2Perf(trace2PerfPath, title) { fs.rmSync(trace2PerfPath, { force: true }); } } +/** + * An empty bare mirror with `git clone --mirror`'s config, for gitfs to + * stream the first job's revision into; gitfs completes the mirror in the + * background, so it is a full mirror by the time the job ends. + */ +function initEmptyMirror(mirrorPath, repoUrl) { + return __awaiter(this, void 0, void 0, function* () { + var _a, _b, _c, _d; + const mirrorDir = path.dirname(mirrorPath); + yield exec.exec('sudo', ['mkdir', '-p', mirrorDir]); + const uid = (_b = (_a = process.getuid) === null || _a === void 0 ? void 0 : _a.call(process)) !== null && _b !== void 0 ? _b : 1000; + const gid = (_d = (_c = process.getgid) === null || _c === void 0 ? void 0 : _c.call(process)) !== null && _d !== void 0 ? _d : 1000; + yield exec.exec('sudo', ['chown', '-R', `${uid}:${gid}`, mirrorDir]); + if (fs.existsSync(mirrorPath)) { + yield fs.promises.rm(mirrorPath, { recursive: true, force: true }); + } + yield exec.exec('git', ['init', '-q', '--bare', mirrorPath]); + yield exec.exec('git', [ + '-C', + mirrorPath, + 'remote', + 'add', + '--mirror=fetch', + 'origin', + repoUrl + ]); + }); +} /** * Ensure a bare git mirror exists. If the mirror doesn't exist, clone it. * If the mirror already exists, it is left as-is; the caller brings it up to @@ -3603,6 +3633,7 @@ const stateHelper = __importStar(__nccwpck_require__(4866)); const urlHelper = __importStar(__nccwpck_require__(9437)); const blacksmithCache = __importStar(__nccwpck_require__(9242)); const mirrorTelemetry = __importStar(__nccwpck_require__(7185)); +const gitfs = __importStar(__nccwpck_require__(1574)); const git_command_manager_1 = __nccwpck_require__(738); function getSource(settings) { return __awaiter(this, void 0, void 0, function* () { @@ -3706,6 +3737,8 @@ function getSourceInner(settings, report, onHydrationReport) { // of this run (hydrated now, or synced via ls-remote diff). Only then can // the workspace copy its refs from the mirror instead of the network. let mirrorFresh = false; + // gitfs is hydrating an empty mirror: it has no refs to populate from. + let gitfsHydrates = false; if (blacksmithCache.shouldUseBlacksmithCache()) { const setupStart = Date.now(); try { @@ -3731,7 +3764,22 @@ function getSourceInner(settings, report, onHydrationReport) { const cloneStart = Date.now(); let performedHydration = false; try { - performedHydration = yield blacksmithCache.ensureMirror(cacheInfo.mirrorPath, repositoryUrl, settings.authToken, settings.verbose); + if (!mirrorExisted && + gitfs.isGitfsRequested() && + !gitfs.gitfsIneligibilityReason(settings, { + mirrorPath: cacheInfo.mirrorPath + }) && + (yield gitfs.findBinary())) { + // No blocking clone: gitfs streams the job's revision into the + // empty mirror and completes it in the background. + core.info(`[gitfs] no mirror at ${cacheInfo.mirrorPath}: gitfs hydrates it`); + yield blacksmithCache.initEmptyMirror(cacheInfo.mirrorPath, repositoryUrl); + performedHydration = true; + gitfsHydrates = true; + } + else { + performedHydration = yield blacksmithCache.ensureMirror(cacheInfo.mirrorPath, repositoryUrl, settings.authToken, settings.verbose); + } } catch (error) { if (!mirrorExisted) { @@ -3766,7 +3814,7 @@ function getSourceInner(settings, report, onHydrationReport) { } if (performedHydration) { // A freshly-cloned mirror is exactly the remote's current state - mirrorFresh = true; + mirrorFresh = !gitfsHydrates; } else if (settings.fetchDepth <= 0) { // Bring the mirror's branch/tag refs up to date with the remote @@ -3810,6 +3858,23 @@ function getSourceInner(settings, report, onHydrationReport) { cacheInfo = null; } } + // Serve the workspace with gitfs when the job asks for it: mount the + // requested revision from the mirror now (milliseconds), and let the + // tree be written out behind the job. The .git set up below is created + // through the mount, so git works from the first step. Anything gitfs + // cannot serve falls through to the regular fetch + checkout. + let gitfsRevision = null; + if (gitfs.isGitfsRequested()) { + core.startGroup('Mounting the repository with gitfs'); + try { + gitfsRevision = yield mountWithGitfs(settings, cacheInfo, repositoryUrl); + } + catch (error) { + core.warning(`[gitfs] mount failed, using the regular checkout: ${error}`); + gitfsRevision = null; + } + core.endGroup(); + } // Initialize the repository if (!fsHelper.directoryExistsSync(path.join(settings.repositoryPath, '.git'))) { core.startGroup('Initializing the repository'); @@ -3854,157 +3919,21 @@ function getSourceInner(settings, report, onHydrationReport) { if (settings.lfs) { yield git.lfsInstall(); } - // Fetch - core.startGroup('Fetching the repository'); - const objectsDir = path.join(settings.repositoryPath, '.git', 'objects'); - const objectsBytesBefore = cacheInfo - ? yield mirrorTelemetry.dirSizeBytesOrNull(objectsDir) - : null; - const fetchStart = Date.now(); - const fetchOptions = {}; - if (settings.filter) { - fetchOptions.filter = settings.filter; - } - else if (settings.sparseCheckout) { - fetchOptions.filter = 'blob:none'; - } - if (settings.fetchDepth <= 0) { - // When the Blacksmith mirror is available and synced with the remote, - // copy branch/tag refs from it locally (objects are already shared via - // alternates) instead of doing a full +refs/heads/* fetch over the - // network, and only ask the network for the specific ref/commit this - // run needs. - let fetchedFromMirror = false; - if (cacheInfo && - mirrorFresh && - !fetchOptions.filter && - !fsHelper.fileExistsSync(path.join(settings.repositoryPath, '.git', 'shallow'))) { - fetchedFromMirror = yield blacksmithCache.fetchRefsFromMirror(settings.repositoryPath, cacheInfo.mirrorPath, git.getEnvironment()); - } - if (fetchedFromMirror) { - // The mirror copy only materializes branches and tags. Any other ref - // (refs/pull/*, etc.) must be fetched from the network so its local - // destination ref exists for checkout. For branches/tags, verify the - // wanted ref/commit is present and fetch it directly if not. - const upperRef = (settings.ref || '').toUpperCase(); - const coveredByMirror = !settings.ref || - !upperRef.startsWith('REFS/') || - upperRef.startsWith('REFS/HEADS/') || - upperRef.startsWith('REFS/TAGS/'); - let verified = false; - if (coveredByMirror) { - verified = yield refHelper.testRef(git, settings.ref, settings.commit); - if (verified && settings.commit) { - verified = yield git.shaExists(settings.commit); - } - } - if (!verified) { - const refSpec = refHelper.getRefSpec(settings.ref, settings.commit); - yield git.fetch(refSpec, fetchOptions); - } - } - else { - // Fetch all branches and tags - let refSpec = refHelper.getRefSpecForAllHistory(settings.ref, settings.commit); - yield git.fetch(refSpec, fetchOptions); - // When all history is fetched, the ref we're interested in may have moved to a different - // commit (push or force push). If so, fetch again with a targeted refspec. - if (!(yield refHelper.testRef(git, settings.ref, settings.commit))) { - refSpec = refHelper.getRefSpec(settings.ref, settings.commit); - yield git.fetch(refSpec, fetchOptions); - } - } - } - else { - fetchOptions.fetchDepth = settings.fetchDepth; - fetchOptions.fetchTags = settings.fetchTags; - // With the mirror attached as an alternate, git would treat every - // mirror ref as a known tip: the deepening fetch's connectivity check - // walks the whole mirror object graph and negotiation offers every - // mirror ref. Turn alternate ref discovery off and offer a few mirror - // tips explicitly instead, so the server still sends only the delta - // from the mirror (see resolveShallowNegotiationTips). - if (cacheInfo && - (yield git.version()).checkMinimum(git_command_manager_1.MinimumGitAlternateRefsCommandVersion) && - (yield blacksmithCache.sharesMirrorObjects(settings.repositoryPath, cacheInfo.mirrorPath))) { - fetchOptions.ignoreAlternateRefs = true; - fetchOptions.negotiationTips = - yield blacksmithCache.resolveShallowNegotiationTips(cacheInfo.mirrorPath, settings.ref, process.env['GITHUB_BASE_REF'] || ''); - core.info(`[git-mirror] Shallow fetch negotiating from ${fetchOptions.negotiationTips.length} mirror tip(s) with alternate ref discovery disabled`); - } - const refSpec = refHelper.getRefSpec(settings.ref, settings.commit); - yield git.fetch(refSpec, fetchOptions); - } - if (cacheInfo) { - // Mirror-served: the fetch pulls only the delta the mirror is missing. - report.delta_fetch_ms = Date.now() - fetchStart; - const objectsBytesAfter = yield mirrorTelemetry.dirSizeBytesOrNull(objectsDir); - report.delta_fetch_bytes = - objectsBytesBefore !== null && objectsBytesAfter !== null - ? Math.max(0, objectsBytesAfter - objectsBytesBefore) - : 0; + if (gitfsRevision && cacheInfo) { + // The tree is already served from the mirror: give the workspace the + // refs and index the regular path would have produced and skip the + // fetch + checkout. + core.startGroup('Pointing the repository at the mounted revision'); + const pointStart = Date.now(); + yield blacksmithCache.fetchRefsFromMirror(settings.repositoryPath, cacheInfo.mirrorPath, git.getEnvironment()); + yield gitfs.pointWorkspaceAt(settings.repositoryPath, settings.ref, gitfsRevision, git.getEnvironment()); + report.clone_from_mirror_ms += Date.now() - pointStart; report.mirror_size_bytes = yield mirrorTelemetry.packSizeBytes(cacheInfo.mirrorPath); report.ref_count = yield mirrorTelemetry.refCount(cacheInfo.mirrorPath); - } - else { - // Fallback/bypass: the fetch is the full checkout cost. - report.full_checkout_ms = Date.now() - fetchStart; - } - core.endGroup(); - // Checkout info - core.startGroup('Determining the checkout info'); - const checkoutInfo = yield refHelper.getCheckoutInfo(git, settings.ref, settings.commit); - core.endGroup(); - // LFS fetch - // Explicit lfs-fetch to avoid slow checkout (fetches one lfs object at a time). - // Explicit lfs fetch will fetch lfs objects in parallel. - // For sparse checkouts, let `checkout` fetch the needed objects lazily. - if (settings.lfs && !settings.sparseCheckout) { - core.startGroup('Fetching LFS objects'); - const lfsStart = Date.now(); - yield git.lfsFetch(checkoutInfo.startPoint || checkoutInfo.ref); - report.lfs_ms = Date.now() - lfsStart; core.endGroup(); } - // Sparse checkout - if (!settings.sparseCheckout) { - let gitVersion = yield git.version(); - // no need to disable sparse-checkout if the installed git runtime doesn't even support it. - if (gitVersion.checkMinimum(git_command_manager_1.MinimumGitSparseCheckoutVersion)) { - yield git.disableSparseCheckout(); - } - } else { - core.startGroup('Setting up sparse checkout'); - if (settings.sparseCheckoutConeMode) { - yield git.sparseCheckout(settings.sparseCheckout); - } - else { - yield git.sparseCheckoutNonConeMode(settings.sparseCheckout); - } - core.endGroup(); - } - // Checkout - core.startGroup('Checking out the ref'); - const checkoutStart = Date.now(); - yield git.checkout(checkoutInfo.ref, checkoutInfo.startPoint); - if (cacheInfo) { - // Materializing the working tree reads objects from the mirror through - // alternates — the second half of the clone-from-mirror cost. - report.clone_from_mirror_ms += Date.now() - checkoutStart; - } - else { - // Fallback/bypass: working-tree materialization is part of the full - // checkout cost, keeping phase totals comparable across serving modes. - report.full_checkout_ms += Date.now() - checkoutStart; - } - core.endGroup(); - // Dissociate from Blacksmith mirror if requested - // This copies all objects from alternates into the local repo so it's independent - if (settings.dissociate && cacheInfo) { - core.startGroup('Dissociating from Blacksmith mirror'); - yield blacksmithCache.dissociate(settings.repositoryPath, git.getEnvironment()); - core.endGroup(); + yield fetchAndCheckout(settings, git, cacheInfo, mirrorFresh, report); } // Submodules if (settings.submodules) { @@ -4048,6 +3977,210 @@ function getSourceInner(settings, report, onHydrationReport) { } }); } +/** + * Mount the requested revision from the mirror with gitfs. Returns null, + * after logging why, when this checkout is outside the gitfs cohort; throws + * when gitfs was attempted and failed. + */ +function mountWithGitfs(settings, cacheInfo, repositoryUrl) { + return __awaiter(this, void 0, void 0, function* () { + var _a; + const reason = gitfs.gitfsIneligibilityReason(settings, { + mirrorPath: (_a = cacheInfo === null || cacheInfo === void 0 ? void 0 : cacheInfo.mirrorPath) !== null && _a !== void 0 ? _a : null + }); + if (reason || !cacheInfo) { + core.info(`[gitfs] not used: ${reason}`); + return null; + } + const binary = yield gitfs.findBinary(); + if (!binary) { + core.info('[gitfs] not used: gitfs binary not found on PATH'); + return null; + } + const candidates = gitfs.mirrorRevisionCandidates(settings.ref, settings.commit); + const revision = yield gitfs.resolveInMirror(cacheInfo.mirrorPath, candidates); + const mountStart = Date.now(); + if (revision) { + yield gitfs.mount(binary, cacheInfo.mirrorPath, revision.sha, settings.repositoryPath); + stateHelper.setGitfsMountPoint(settings.repositoryPath); + core.info(`[gitfs] ${settings.repositoryPath} serves ${revision.sha} from the mirror (mounted in ${Date.now() - mountStart} ms); the tree is written out in the background`); + return revision; + } + // The mirror lacks the revision (an empty mirror, a push it has not + // synced, a pull-request merge ref): gitfs fetches it from the remote into + // the mirror and serves the pack as it arrives; the rest of the repository + // follows in the background. The mirror changes, so the post step commits + // it. + const rev = gitfs.coldRevision(settings.ref, settings.commit); + core.info(`[gitfs] ${rev} is not in the mirror, streaming it from the remote`); + const sha = yield gitfs.mountCold(binary, cacheInfo.mirrorPath, rev, settings.repositoryPath, repositoryUrl, settings.authToken); + stateHelper.setGitfsMountPoint(settings.repositoryPath); + stateHelper.setBlacksmithCacheMirrorChanged(true); + const { mirrorRef, kind } = candidates[0]; + core.info(`[gitfs] ${settings.repositoryPath} serves ${sha} streamed from the remote (mounted in ${Date.now() - mountStart} ms); the mirror is completed in the background`); + return { sha, mirrorRef, kind }; + }); +} +/** + * The regular path: fetch the requested revision into the workspace (from + * the mirror where possible) and check it out. + */ +function fetchAndCheckout(settings, git, cacheInfo, mirrorFresh, report) { + return __awaiter(this, void 0, void 0, function* () { + // Fetch + core.startGroup('Fetching the repository'); + const objectsDir = path.join(settings.repositoryPath, '.git', 'objects'); + const objectsBytesBefore = cacheInfo + ? yield mirrorTelemetry.dirSizeBytesOrNull(objectsDir) + : null; + const fetchStart = Date.now(); + const fetchOptions = {}; + if (settings.filter) { + fetchOptions.filter = settings.filter; + } + else if (settings.sparseCheckout) { + fetchOptions.filter = 'blob:none'; + } + if (settings.fetchDepth <= 0) { + // When the Blacksmith mirror is available and synced with the remote, + // copy branch/tag refs from it locally (objects are already shared via + // alternates) instead of doing a full +refs/heads/* fetch over the + // network, and only ask the network for the specific ref/commit this + // run needs. + let fetchedFromMirror = false; + if (cacheInfo && + mirrorFresh && + !fetchOptions.filter && + !fsHelper.fileExistsSync(path.join(settings.repositoryPath, '.git', 'shallow'))) { + fetchedFromMirror = yield blacksmithCache.fetchRefsFromMirror(settings.repositoryPath, cacheInfo.mirrorPath, git.getEnvironment()); + } + if (fetchedFromMirror) { + // The mirror copy only materializes branches and tags. Any other ref + // (refs/pull/*, etc.) must be fetched from the network so its local + // destination ref exists for checkout. For branches/tags, verify the + // wanted ref/commit is present and fetch it directly if not. + const upperRef = (settings.ref || '').toUpperCase(); + const coveredByMirror = !settings.ref || + !upperRef.startsWith('REFS/') || + upperRef.startsWith('REFS/HEADS/') || + upperRef.startsWith('REFS/TAGS/'); + let verified = false; + if (coveredByMirror) { + verified = yield refHelper.testRef(git, settings.ref, settings.commit); + if (verified && settings.commit) { + verified = yield git.shaExists(settings.commit); + } + } + if (!verified) { + const refSpec = refHelper.getRefSpec(settings.ref, settings.commit); + yield git.fetch(refSpec, fetchOptions); + } + } + else { + // Fetch all branches and tags + let refSpec = refHelper.getRefSpecForAllHistory(settings.ref, settings.commit); + yield git.fetch(refSpec, fetchOptions); + // When all history is fetched, the ref we're interested in may have moved to a different + // commit (push or force push). If so, fetch again with a targeted refspec. + if (!(yield refHelper.testRef(git, settings.ref, settings.commit))) { + refSpec = refHelper.getRefSpec(settings.ref, settings.commit); + yield git.fetch(refSpec, fetchOptions); + } + } + } + else { + fetchOptions.fetchDepth = settings.fetchDepth; + fetchOptions.fetchTags = settings.fetchTags; + // With the mirror attached as an alternate, git would treat every + // mirror ref as a known tip: the deepening fetch's connectivity check + // walks the whole mirror object graph and negotiation offers every + // mirror ref. Turn alternate ref discovery off and offer a few mirror + // tips explicitly instead, so the server still sends only the delta + // from the mirror (see resolveShallowNegotiationTips). + if (cacheInfo && + (yield git.version()).checkMinimum(git_command_manager_1.MinimumGitAlternateRefsCommandVersion) && + (yield blacksmithCache.sharesMirrorObjects(settings.repositoryPath, cacheInfo.mirrorPath))) { + fetchOptions.ignoreAlternateRefs = true; + fetchOptions.negotiationTips = + yield blacksmithCache.resolveShallowNegotiationTips(cacheInfo.mirrorPath, settings.ref, process.env['GITHUB_BASE_REF'] || ''); + core.info(`[git-mirror] Shallow fetch negotiating from ${fetchOptions.negotiationTips.length} mirror tip(s) with alternate ref discovery disabled`); + } + const refSpec = refHelper.getRefSpec(settings.ref, settings.commit); + yield git.fetch(refSpec, fetchOptions); + } + if (cacheInfo) { + // Mirror-served: the fetch pulls only the delta the mirror is missing. + report.delta_fetch_ms = Date.now() - fetchStart; + const objectsBytesAfter = yield mirrorTelemetry.dirSizeBytesOrNull(objectsDir); + report.delta_fetch_bytes = + objectsBytesBefore !== null && objectsBytesAfter !== null + ? Math.max(0, objectsBytesAfter - objectsBytesBefore) + : 0; + report.mirror_size_bytes = yield mirrorTelemetry.packSizeBytes(cacheInfo.mirrorPath); + report.ref_count = yield mirrorTelemetry.refCount(cacheInfo.mirrorPath); + } + else { + // Fallback/bypass: the fetch is the full checkout cost. + report.full_checkout_ms = Date.now() - fetchStart; + } + core.endGroup(); + // Checkout info + core.startGroup('Determining the checkout info'); + const checkoutInfo = yield refHelper.getCheckoutInfo(git, settings.ref, settings.commit); + core.endGroup(); + // LFS fetch + // Explicit lfs-fetch to avoid slow checkout (fetches one lfs object at a time). + // Explicit lfs fetch will fetch lfs objects in parallel. + // For sparse checkouts, let `checkout` fetch the needed objects lazily. + if (settings.lfs && !settings.sparseCheckout) { + core.startGroup('Fetching LFS objects'); + const lfsStart = Date.now(); + yield git.lfsFetch(checkoutInfo.startPoint || checkoutInfo.ref); + report.lfs_ms = Date.now() - lfsStart; + core.endGroup(); + } + // Sparse checkout + if (!settings.sparseCheckout) { + let gitVersion = yield git.version(); + // no need to disable sparse-checkout if the installed git runtime doesn't even support it. + if (gitVersion.checkMinimum(git_command_manager_1.MinimumGitSparseCheckoutVersion)) { + yield git.disableSparseCheckout(); + } + } + else { + core.startGroup('Setting up sparse checkout'); + if (settings.sparseCheckoutConeMode) { + yield git.sparseCheckout(settings.sparseCheckout); + } + else { + yield git.sparseCheckoutNonConeMode(settings.sparseCheckout); + } + core.endGroup(); + } + // Checkout + core.startGroup('Checking out the ref'); + const checkoutStart = Date.now(); + yield git.checkout(checkoutInfo.ref, checkoutInfo.startPoint); + if (cacheInfo) { + // Materializing the working tree reads objects from the mirror through + // alternates — the second half of the clone-from-mirror cost. + report.clone_from_mirror_ms += Date.now() - checkoutStart; + } + else { + // Fallback/bypass: working-tree materialization is part of the full + // checkout cost, keeping phase totals comparable across serving modes. + report.full_checkout_ms += Date.now() - checkoutStart; + } + core.endGroup(); + // Dissociate from Blacksmith mirror if requested + // This copies all objects from alternates into the local repo so it's independent + if (settings.dissociate && cacheInfo) { + core.startGroup('Dissociating from Blacksmith mirror'); + yield blacksmithCache.dissociate(settings.repositoryPath, git.getEnvironment()); + core.endGroup(); + } + }); +} function cleanup(repositoryPath) { return __awaiter(this, void 0, void 0, function* () { // Repo exists? @@ -4181,6 +4314,376 @@ class GitVersion { exports.GitVersion = GitVersion; +/***/ }), + +/***/ 1574: +/***/ (function(__unused_webpack_module, exports, __nccwpck_require__) { + +"use strict"; + +var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) { + if (k2 === undefined) k2 = k; + var desc = Object.getOwnPropertyDescriptor(m, k); + if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) { + desc = { enumerable: true, get: function() { return m[k]; } }; + } + Object.defineProperty(o, k2, desc); +}) : (function(o, m, k, k2) { + if (k2 === undefined) k2 = k; + o[k2] = m[k]; +})); +var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) { + Object.defineProperty(o, "default", { enumerable: true, value: v }); +}) : function(o, v) { + o["default"] = v; +}); +var __importStar = (this && this.__importStar) || function (mod) { + if (mod && mod.__esModule) return mod; + var result = {}; + if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k); + __setModuleDefault(result, mod); + return result; +}; +var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) { + function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); } + return new (P || (P = Promise))(function (resolve, reject) { + function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } } + function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } } + function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); } + step((generator = generator.apply(thisArg, _arguments || [])).next()); + }); +}; +Object.defineProperty(exports, "__esModule", ({ value: true })); +exports.GITFS_ENV = void 0; +exports.isGitfsRequested = isGitfsRequested; +exports.gitfsIneligibilityReason = gitfsIneligibilityReason; +exports.findBinary = findBinary; +exports.mirrorRevisionCandidates = mirrorRevisionCandidates; +exports.coldRevision = coldRevision; +exports.resolveInMirror = resolveInMirror; +exports.fetchIntoMirror = fetchIntoMirror; +exports.mount = mount; +exports.mountCold = mountCold; +exports.pointWorkspaceAt = pointWorkspaceAt; +exports.teardown = teardown; +const core = __importStar(__nccwpck_require__(2186)); +const exec = __importStar(__nccwpck_require__(1514)); +const fs = __importStar(__nccwpck_require__(7147)); +const io = __importStar(__nccwpck_require__(7436)); +const path = __importStar(__nccwpck_require__(1017)); +const container_detector_1 = __nccwpck_require__(6424); +const blacksmith_cache_1 = __nccwpck_require__(9242); +/** + * gitfs serves the workspace from the git mirror sticky disk instead of + * checking it out: `gitfs mount --ref --daemon` + * returns as soon as the tree is mounted (milliseconds), writes the real + * checkout out in the background and swaps it in for the mount once it is + * complete. Writes made through the mount in the meantime (`.git`, npm's + * node_modules, ...) land in that same tree. The agent enables it per job via + * BLACKSMITH_USE_GITFS=true, driven by the installation's use_gitfs_checkout + * flag; anything gitfs cannot serve falls through to the regular checkout. + */ +exports.GITFS_ENV = 'BLACKSMITH_USE_GITFS'; +const GITFS_BINARY = 'gitfs'; +const DAEMON_EXIT_TIMEOUT_MS = 10000; +const DAEMON_KILL_TIMEOUT_MS = 5000; +function isGitfsRequested(env = process.env) { + return env[exports.GITFS_ENV] === 'true'; +} +/** + * Why gitfs cannot serve this checkout, or null when it can. The v1 cohort is + * the plain case: a Linux runner VM with the mirror mounted, a branch, tag, + * pull ref or commit to serve, and none of the features that need the + * workspace to be a real fetch (submodules, LFS, sparse/partial clones, + * dissociate). + */ +function gitfsIneligibilityReason(settings, ctx) { + var _a, _b; + const platform = (_a = ctx.platform) !== null && _a !== void 0 ? _a : process.platform; + if (platform !== 'linux') { + return `platform ${platform} is not linux`; + } + if (!ctx.mirrorPath) { + return 'git mirror is not available'; + } + const inContainer = (_b = ctx.inContainer) !== null && _b !== void 0 ? _b : (0, container_detector_1.isRunningInContainer)(); + if (inContainer) { + return 'running inside a container'; + } + if (!settings.ref && !settings.commit) { + return 'no ref or commit requested'; + } + if (settings.submodules) { + return 'submodules are enabled'; + } + if (settings.lfs) { + return 'lfs is enabled'; + } + if (settings.sparseCheckout && settings.sparseCheckout.length > 0) { + return 'sparse checkout is enabled'; + } + if (settings.filter) { + return `filter ${settings.filter} is set`; + } + if (settings.dissociate) { + return 'dissociate is enabled'; + } + if (fs.existsSync(path.join(settings.repositoryPath, '.git'))) { + return 'workspace already contains a repository'; + } + return null; +} +function findBinary() { + return __awaiter(this, void 0, void 0, function* () { + const found = yield io.which(GITFS_BINARY, false); + return found || null; + }); +} +function classify(ref) { + const upper = ref.toUpperCase(); + if (upper.startsWith('REFS/HEADS/')) + return 'branch'; + if (upper.startsWith('REFS/TAGS/')) + return 'tag'; + if (upper.startsWith('REFS/PULL/')) + return 'pull'; + return 'other'; +} +/** + * Revisions to try in the mirror for the requested ref/commit, in order. A + * commit wins over the ref (that is what the checkout would end up at); an + * unqualified ref is a branch first, then a tag, as in getCheckoutInfo. + */ +function mirrorRevisionCandidates(ref, commit) { + if (commit) { + const kind = ref ? classify(ref) : 'commit'; + const mirrorRef = ref && ref.toUpperCase().startsWith('REFS/') ? ref : ''; + return [{ rev: `${commit}^{commit}`, mirrorRef, kind }]; + } + if (!ref) { + return []; + } + if (ref.toUpperCase().startsWith('REFS/')) { + return [{ rev: `${ref}^{commit}`, mirrorRef: ref, kind: classify(ref) }]; + } + return [ + { + rev: `refs/heads/${ref}^{commit}`, + mirrorRef: `refs/heads/${ref}`, + kind: 'branch' + }, + { + rev: `refs/tags/${ref}^{commit}`, + mirrorRef: `refs/tags/${ref}`, + kind: 'tag' + } + ]; +} +/** + * What `gitfs mount --url` fetches for a revision the mirror lacks: a commit + * by sha, a qualified ref by name, an unqualified ref as a branch. + */ +function coldRevision(ref, commit) { + if (commit) { + return commit; + } + if (ref.toUpperCase().startsWith('REFS/')) { + return ref; + } + return `refs/heads/${ref}`; +} +function resolveInMirror(mirrorPath, candidates) { + return __awaiter(this, void 0, void 0, function* () { + for (const candidate of candidates) { + const result = yield exec.getExecOutput('git', ['-C', mirrorPath, 'rev-parse', '--verify', '--quiet', candidate.rev], { silent: true, ignoreReturnCode: true }); + const sha = result.stdout.trim(); + if (result.exitCode === 0 && /^[0-9a-f]{40}$/.test(sha)) { + return { sha, mirrorRef: candidate.mirrorRef, kind: candidate.kind }; + } + } + return null; + }); +} +function fetchIntoMirror(mirrorPath, repositoryUrl, authToken, refspecs) { + return __awaiter(this, void 0, void 0, function* () { + const auth = (0, blacksmith_cache_1.getAuthConfigArgs)(repositoryUrl, authToken); + yield exec.exec('git', [ + '-C', + mirrorPath, + '-c', + `${auth.configKey}=${auth.configValue}`, + '-c', + 'gc.auto=0', + 'fetch', + '--no-tags', + '--no-recurse-submodules', + '--no-write-fetch-head', + repositoryUrl, + ...refspecs + ]); + }); +} +function mount(binary, mirrorPath, sha, mountpoint) { + return __awaiter(this, void 0, void 0, function* () { + if (fs.existsSync(mountpoint) && fs.readdirSync(mountpoint).length > 0) { + throw new Error(`${mountpoint} is not empty`); + } + yield exec.exec(binary, [ + 'mount', + mirrorPath, + '--ref', + sha, + mountpoint, + '--daemon' + ]); + }); +} +/** + * Mount a revision the mirror does not have: gitfs fetches it from the + * remote into the mirror (`--depth 1`, serving the pack as it arrives) and + * completes the mirror in the background. Returns the commit sha, which + * gitfs prints once the commit is in, before git has written the ref. + */ +function mountCold(binary, mirrorPath, rev, mountpoint, repositoryUrl, authToken) { + return __awaiter(this, void 0, void 0, function* () { + var _a, _b; + if (fs.existsSync(mountpoint) && fs.readdirSync(mountpoint).length > 0) { + throw new Error(`${mountpoint} is not empty`); + } + const env = {}; + for (const [key, value] of Object.entries(process.env)) { + if (value !== undefined) { + env[key] = value; + } + } + env.GITFS_TOKEN = authToken; + const result = yield exec.getExecOutput(binary, [ + 'mount', + mirrorPath, + '--ref', + rev, + mountpoint, + '--daemon', + '--url', + repositoryUrl + ], { env }); + const sha = (_b = (_a = result.stdout.trim().split('\n').pop()) === null || _a === void 0 ? void 0 : _a.trim()) !== null && _b !== void 0 ? _b : ''; + if (!/^[0-9a-f]{40}$/.test(sha)) { + throw new Error(`gitfs did not report the mounted commit: ${result.stdout.trim()}`); + } + return sha; + }); +} +/** + * Point the workspace at the mounted revision without touching the tree: + * the local branch / tag / pull ref the regular checkout would have created, + * HEAD on it (or detached, as `git checkout` would leave it), and an index + * for the tree. The files are already there, served by gitfs. + */ +function pointWorkspaceAt(repositoryPath, ref, revision, env) { + return __awaiter(this, void 0, void 0, function* () { + const git = (...args) => __awaiter(this, void 0, void 0, function* () { + yield exec.exec('git', ['-C', repositoryPath, ...args], { env }); + }); + const { sha } = revision; + if (revision.kind === 'branch') { + const branch = revision.mirrorRef.slice('refs/heads/'.length); + yield git('update-ref', `refs/remotes/origin/${branch}`, sha); + yield git('update-ref', `refs/heads/${branch}`, sha); + yield git('config', `branch.${branch}.remote`, 'origin'); + yield git('config', `branch.${branch}.merge`, `refs/heads/${branch}`); + yield git('symbolic-ref', 'HEAD', `refs/heads/${branch}`); + } + else { + if (revision.kind === 'tag') { + yield git('update-ref', revision.mirrorRef, sha); + } + else if (revision.kind === 'pull') { + const name = revision.mirrorRef.slice('refs/pull/'.length); + yield git('update-ref', `refs/remotes/pull/${name}`, sha); + } + yield git('update-ref', '--no-deref', 'HEAD', sha); + } + // Index entries without stat data: git re-verifies file contents on the + // first status/diff instead of touching every file now. + yield git('read-tree', sha); + }); +} +function daemonPids(mirrorPath) { + const pids = []; + let entries; + try { + entries = fs.readdirSync('/proc'); + } + catch (_a) { + return pids; + } + for (const entry of entries) { + if (!/^\d+$/.test(entry)) + continue; + let cmdline; + try { + cmdline = fs.readFileSync(`/proc/${entry}/cmdline`, 'utf8'); + } + catch (_b) { + continue; + } + const argv = cmdline.split('\0'); + if (path.basename(argv[0] || '') === GITFS_BINARY && + argv.includes('mount') && + argv.includes(mirrorPath)) { + pids.push(Number(entry)); + } + } + return pids; +} +function waitForDaemons(mirrorPath, timeoutMs) { + return __awaiter(this, void 0, void 0, function* () { + const deadline = Date.now() + timeoutMs; + let pids = daemonPids(mirrorPath); + while (pids.length > 0 && Date.now() < deadline) { + yield new Promise(resolve => setTimeout(resolve, 100)); + pids = daemonPids(mirrorPath); + } + return pids; + }); +} +/** + * Take gitfs down before the mirror sticky disk is unmounted: the daemon has + * the mirror's packs mapped, so the disk cannot be released while it lives. + * `gitfs unmount` handles both phases — while the tree is still being + * written the workspace path is a symlink into the mount and unmounting + * discards the partial tree (the workspace is left empty); after the swap + * the path is a plain directory and the FUSE view that served it is still up + * on the hidden sibling path, which unmount takes down. + */ +function teardown(mountpoint, mirrorPath) { + return __awaiter(this, void 0, void 0, function* () { + const binary = yield findBinary(); + if (binary && fs.existsSync(mountpoint)) { + core.info(`[gitfs] Unmounting ${mountpoint}`); + yield exec.exec(binary, ['unmount', mountpoint], { ignoreReturnCode: true }); + } + let pids = yield waitForDaemons(mirrorPath, DAEMON_EXIT_TIMEOUT_MS); + if (pids.length > 0) { + core.info(`[gitfs] Stopping daemon(s) ${pids.join(', ')} still serving ${mirrorPath}`); + for (const pid of pids) { + try { + process.kill(pid, 'SIGTERM'); + } + catch (_a) { + // already gone + } + } + pids = yield waitForDaemons(mirrorPath, DAEMON_KILL_TIMEOUT_MS); + if (pids.length > 0) { + core.warning(`[gitfs] daemon(s) ${pids.join(', ')} did not exit; the mirror unmount may fail`); + } + } + }); +} + + /***/ }), /***/ 138: @@ -4691,6 +5194,7 @@ const inputHelper = __importStar(__nccwpck_require__(5480)); const path = __importStar(__nccwpck_require__(1017)); const stateHelper = __importStar(__nccwpck_require__(4866)); const blacksmithCache = __importStar(__nccwpck_require__(9242)); +const gitfs = __importStar(__nccwpck_require__(1574)); const step_checker_1 = __nccwpck_require__(9716); const internal_metrics_1 = __nccwpck_require__(7753); const mirrorTelemetry = __importStar(__nccwpck_require__(7185)); @@ -4731,6 +5235,16 @@ function cleanup() { const repoName = stateHelper.BlacksmithCacheRepoName; const mountPoint = stateHelper.BlacksmithCacheMountPoint; const mirrorPath = stateHelper.BlacksmithCacheMirrorPath; + // gitfs maps the mirror's packs, so it has to be gone before the sticky + // disk can be unmounted below. + if (stateHelper.GitfsMountPoint && mirrorPath) { + try { + yield gitfs.teardown(stateHelper.GitfsMountPoint, mirrorPath); + } + catch (error) { + core.warning(`[gitfs] teardown failed: ${error}`); + } + } const performedHydration = stateHelper.BlacksmithCachePerformedHydration; let mirrorChanged = stateHelper.BlacksmithCacheMirrorChanged; let mirrorSyncFailed = stateHelper.BlacksmithCacheMirrorSyncFailed; @@ -5524,7 +6038,7 @@ var __importStar = (this && this.__importStar) || function (mod) { return result; }; Object.defineProperty(exports, "__esModule", ({ value: true })); -exports.BlacksmithCacheCommitEarlyDenyReason = exports.BlacksmithCacheVerbose = exports.BlacksmithCacheRepoUrl = exports.BlacksmithCacheMirrorSyncDeferred = exports.BlacksmithCacheMirrorSyncTimedOut = exports.BlacksmithCacheMirrorSyncFailed = exports.BlacksmithCacheMirrorChanged = exports.BlacksmithCachePerformedHydration = exports.BlacksmithCacheStickyDiskKey = exports.BlacksmithCacheRepoName = exports.BlacksmithCacheMountPoint = exports.BlacksmithCacheMirrorPath = exports.BlacksmithCacheExposeId = exports.SshKnownHostsPath = exports.SshKeyPath = exports.PostSetSafeDirectory = exports.RepositoryPath = exports.IsPost = void 0; +exports.BlacksmithCacheCommitEarlyDenyReason = exports.GitfsMountPoint = exports.BlacksmithCacheVerbose = exports.BlacksmithCacheRepoUrl = exports.BlacksmithCacheMirrorSyncDeferred = exports.BlacksmithCacheMirrorSyncTimedOut = exports.BlacksmithCacheMirrorSyncFailed = exports.BlacksmithCacheMirrorChanged = exports.BlacksmithCachePerformedHydration = exports.BlacksmithCacheStickyDiskKey = exports.BlacksmithCacheRepoName = exports.BlacksmithCacheMountPoint = exports.BlacksmithCacheMirrorPath = exports.BlacksmithCacheExposeId = exports.SshKnownHostsPath = exports.SshKeyPath = exports.PostSetSafeDirectory = exports.RepositoryPath = exports.IsPost = void 0; exports.setRepositoryPath = setRepositoryPath; exports.setSshKeyPath = setSshKeyPath; exports.setSshKnownHostsPath = setSshKnownHostsPath; @@ -5542,6 +6056,7 @@ exports.setBlacksmithCacheMirrorSyncDeferred = setBlacksmithCacheMirrorSyncDefer exports.setBlacksmithCacheRepoUrl = setBlacksmithCacheRepoUrl; exports.setBlacksmithCacheVerbose = setBlacksmithCacheVerbose; exports.setBlacksmithCacheCommitEarlyDenyReason = setBlacksmithCacheCommitEarlyDenyReason; +exports.setGitfsMountPoint = setGitfsMountPoint; const core = __importStar(__nccwpck_require__(2186)); /** * Indicates whether the POST action is running @@ -5616,6 +6131,11 @@ exports.BlacksmithCacheVerbose = core.getState('blacksmithCacheVerbose') === 'tr * Reason the host gave at mount time for denying this job's sticky disk commit * (e.g. branch protection). Empty when no denial was reported. */ +/** + * Workspace path served by gitfs, so the POST action can take the mount down + * before the mirror sticky disk is unmounted. + */ +exports.GitfsMountPoint = core.getState('gitfsMountPoint'); exports.BlacksmithCacheCommitEarlyDenyReason = core.getState('blacksmithCacheCommitEarlyDenyReason'); /** * Save the repository path so the POST action can retrieve the value. @@ -5718,6 +6238,9 @@ function setBlacksmithCacheVerbose(verbose) { function setBlacksmithCacheCommitEarlyDenyReason(reason) { core.saveState('blacksmithCacheCommitEarlyDenyReason', reason); } +function setGitfsMountPoint(mountPoint) { + core.saveState('gitfsMountPoint', mountPoint); +} // Publish a variable so that when the POST action runs, it can determine it should run the cleanup logic. // This is necessary since we don't have a separate entry point. if (!exports.IsPost) { diff --git a/src/blacksmith-cache.ts b/src/blacksmith-cache.ts index f073efc..d290f70 100644 --- a/src/blacksmith-cache.ts +++ b/src/blacksmith-cache.ts @@ -431,7 +431,7 @@ export async function setupCache( * 1. The header value is not visible in process arguments * 2. It follows the same pattern used by the upstream checkout action */ -function getAuthConfigArgs( +export function getAuthConfigArgs( repoUrl: string, authToken: string ): {configKey: string; configValue: string} { @@ -548,6 +548,35 @@ function summarizeTrace2Perf(trace2PerfPath: string, title: string): void { } } +/** + * An empty bare mirror with `git clone --mirror`'s config, for gitfs to + * stream the first job's revision into; gitfs completes the mirror in the + * background, so it is a full mirror by the time the job ends. + */ +export async function initEmptyMirror( + mirrorPath: string, + repoUrl: string +): Promise { + const mirrorDir = path.dirname(mirrorPath) + await exec.exec('sudo', ['mkdir', '-p', mirrorDir]) + const uid = process.getuid?.() ?? 1000 + const gid = process.getgid?.() ?? 1000 + await exec.exec('sudo', ['chown', '-R', `${uid}:${gid}`, mirrorDir]) + if (fs.existsSync(mirrorPath)) { + await fs.promises.rm(mirrorPath, {recursive: true, force: true}) + } + await exec.exec('git', ['init', '-q', '--bare', mirrorPath]) + await exec.exec('git', [ + '-C', + mirrorPath, + 'remote', + 'add', + '--mirror=fetch', + 'origin', + repoUrl + ]) +} + /** * Ensure a bare git mirror exists. If the mirror doesn't exist, clone it. * If the mirror already exists, it is left as-is; the caller brings it up to diff --git a/src/git-source-provider.ts b/src/git-source-provider.ts index f4c9d45..0d5faff 100644 --- a/src/git-source-provider.ts +++ b/src/git-source-provider.ts @@ -11,6 +11,7 @@ import * as stateHelper from './state-helper' import * as urlHelper from './url-helper' import * as blacksmithCache from './blacksmith-cache' import * as mirrorTelemetry from './mirror-telemetry' +import * as gitfs from './gitfs' import { FetchOptions, MinimumGitAlternateRefsCommandVersion, @@ -160,6 +161,8 @@ async function getSourceInner( // of this run (hydrated now, or synced via ls-remote diff). Only then can // the workspace copy its refs from the mirror instead of the network. let mirrorFresh = false + // gitfs is hydrating an empty mirror: it has no refs to populate from. + let gitfsHydrates = false if (blacksmithCache.shouldUseBlacksmithCache()) { const setupStart = Date.now() try { @@ -193,12 +196,33 @@ async function getSourceInner( const cloneStart = Date.now() let performedHydration = false try { - performedHydration = await blacksmithCache.ensureMirror( - cacheInfo.mirrorPath, - repositoryUrl, - settings.authToken, - settings.verbose - ) + if ( + !mirrorExisted && + gitfs.isGitfsRequested() && + !gitfs.gitfsIneligibilityReason(settings, { + mirrorPath: cacheInfo.mirrorPath + }) && + (await gitfs.findBinary()) + ) { + // No blocking clone: gitfs streams the job's revision into the + // empty mirror and completes it in the background. + core.info( + `[gitfs] no mirror at ${cacheInfo.mirrorPath}: gitfs hydrates it` + ) + await blacksmithCache.initEmptyMirror( + cacheInfo.mirrorPath, + repositoryUrl + ) + performedHydration = true + gitfsHydrates = true + } else { + performedHydration = await blacksmithCache.ensureMirror( + cacheInfo.mirrorPath, + repositoryUrl, + settings.authToken, + settings.verbose + ) + } } catch (error) { if (!mirrorExisted) { onHydrationReport({ @@ -234,7 +258,7 @@ async function getSourceInner( if (performedHydration) { // A freshly-cloned mirror is exactly the remote's current state - mirrorFresh = true + mirrorFresh = !gitfsHydrates } else if (settings.fetchDepth <= 0) { // Bring the mirror's branch/tag refs up to date with the remote // (ls-remote diff + targeted fetch of only the changed refs), so @@ -295,6 +319,25 @@ async function getSourceInner( } } + // Serve the workspace with gitfs when the job asks for it: mount the + // requested revision from the mirror now (milliseconds), and let the + // tree be written out behind the job. The .git set up below is created + // through the mount, so git works from the first step. Anything gitfs + // cannot serve falls through to the regular fetch + checkout. + let gitfsRevision: gitfs.MirrorRevision | null = null + if (gitfs.isGitfsRequested()) { + core.startGroup('Mounting the repository with gitfs') + try { + gitfsRevision = await mountWithGitfs(settings, cacheInfo, repositoryUrl) + } catch (error) { + core.warning( + `[gitfs] mount failed, using the regular checkout: ${error}` + ) + gitfsRevision = null + } + core.endGroup() + } + // Initialize the repository if ( !fsHelper.directoryExistsSync(path.join(settings.repositoryPath, '.git')) @@ -355,194 +398,31 @@ async function getSourceInner( await git.lfsInstall() } - // Fetch - core.startGroup('Fetching the repository') - const objectsDir = path.join(settings.repositoryPath, '.git', 'objects') - const objectsBytesBefore = cacheInfo - ? await mirrorTelemetry.dirSizeBytesOrNull(objectsDir) - : null - const fetchStart = Date.now() - const fetchOptions: FetchOptions = {} - - if (settings.filter) { - fetchOptions.filter = settings.filter - } else if (settings.sparseCheckout) { - fetchOptions.filter = 'blob:none' - } - - if (settings.fetchDepth <= 0) { - // When the Blacksmith mirror is available and synced with the remote, - // copy branch/tag refs from it locally (objects are already shared via - // alternates) instead of doing a full +refs/heads/* fetch over the - // network, and only ask the network for the specific ref/commit this - // run needs. - let fetchedFromMirror = false - if ( - cacheInfo && - mirrorFresh && - !fetchOptions.filter && - !fsHelper.fileExistsSync( - path.join(settings.repositoryPath, '.git', 'shallow') - ) - ) { - fetchedFromMirror = await blacksmithCache.fetchRefsFromMirror( - settings.repositoryPath, - cacheInfo.mirrorPath, - git.getEnvironment() - ) - } - - if (fetchedFromMirror) { - // The mirror copy only materializes branches and tags. Any other ref - // (refs/pull/*, etc.) must be fetched from the network so its local - // destination ref exists for checkout. For branches/tags, verify the - // wanted ref/commit is present and fetch it directly if not. - const upperRef = (settings.ref || '').toUpperCase() - const coveredByMirror = - !settings.ref || - !upperRef.startsWith('REFS/') || - upperRef.startsWith('REFS/HEADS/') || - upperRef.startsWith('REFS/TAGS/') - let verified = false - if (coveredByMirror) { - verified = await refHelper.testRef(git, settings.ref, settings.commit) - if (verified && settings.commit) { - verified = await git.shaExists(settings.commit) - } - } - if (!verified) { - const refSpec = refHelper.getRefSpec(settings.ref, settings.commit) - await git.fetch(refSpec, fetchOptions) - } - } else { - // Fetch all branches and tags - let refSpec = refHelper.getRefSpecForAllHistory( - settings.ref, - settings.commit - ) - await git.fetch(refSpec, fetchOptions) - - // When all history is fetched, the ref we're interested in may have moved to a different - // commit (push or force push). If so, fetch again with a targeted refspec. - if (!(await refHelper.testRef(git, settings.ref, settings.commit))) { - refSpec = refHelper.getRefSpec(settings.ref, settings.commit) - await git.fetch(refSpec, fetchOptions) - } - } - } else { - fetchOptions.fetchDepth = settings.fetchDepth - fetchOptions.fetchTags = settings.fetchTags - // With the mirror attached as an alternate, git would treat every - // mirror ref as a known tip: the deepening fetch's connectivity check - // walks the whole mirror object graph and negotiation offers every - // mirror ref. Turn alternate ref discovery off and offer a few mirror - // tips explicitly instead, so the server still sends only the delta - // from the mirror (see resolveShallowNegotiationTips). - if ( - cacheInfo && - (await git.version()).checkMinimum( - MinimumGitAlternateRefsCommandVersion - ) && - (await blacksmithCache.sharesMirrorObjects( - settings.repositoryPath, - cacheInfo.mirrorPath - )) - ) { - fetchOptions.ignoreAlternateRefs = true - fetchOptions.negotiationTips = - await blacksmithCache.resolveShallowNegotiationTips( - cacheInfo.mirrorPath, - settings.ref, - process.env['GITHUB_BASE_REF'] || '' - ) - core.info( - `[git-mirror] Shallow fetch negotiating from ${fetchOptions.negotiationTips.length} mirror tip(s) with alternate ref discovery disabled` - ) - } - const refSpec = refHelper.getRefSpec(settings.ref, settings.commit) - await git.fetch(refSpec, fetchOptions) - } - if (cacheInfo) { - // Mirror-served: the fetch pulls only the delta the mirror is missing. - report.delta_fetch_ms = Date.now() - fetchStart - const objectsBytesAfter = - await mirrorTelemetry.dirSizeBytesOrNull(objectsDir) - report.delta_fetch_bytes = - objectsBytesBefore !== null && objectsBytesAfter !== null - ? Math.max(0, objectsBytesAfter - objectsBytesBefore) - : 0 + if (gitfsRevision && cacheInfo) { + // The tree is already served from the mirror: give the workspace the + // refs and index the regular path would have produced and skip the + // fetch + checkout. + core.startGroup('Pointing the repository at the mounted revision') + const pointStart = Date.now() + await blacksmithCache.fetchRefsFromMirror( + settings.repositoryPath, + cacheInfo.mirrorPath, + git.getEnvironment() + ) + await gitfs.pointWorkspaceAt( + settings.repositoryPath, + settings.ref, + gitfsRevision, + git.getEnvironment() + ) + report.clone_from_mirror_ms += Date.now() - pointStart report.mirror_size_bytes = await mirrorTelemetry.packSizeBytes( cacheInfo.mirrorPath ) report.ref_count = await mirrorTelemetry.refCount(cacheInfo.mirrorPath) - } else { - // Fallback/bypass: the fetch is the full checkout cost. - report.full_checkout_ms = Date.now() - fetchStart - } - core.endGroup() - - // Checkout info - core.startGroup('Determining the checkout info') - const checkoutInfo = await refHelper.getCheckoutInfo( - git, - settings.ref, - settings.commit - ) - core.endGroup() - - // LFS fetch - // Explicit lfs-fetch to avoid slow checkout (fetches one lfs object at a time). - // Explicit lfs fetch will fetch lfs objects in parallel. - // For sparse checkouts, let `checkout` fetch the needed objects lazily. - if (settings.lfs && !settings.sparseCheckout) { - core.startGroup('Fetching LFS objects') - const lfsStart = Date.now() - await git.lfsFetch(checkoutInfo.startPoint || checkoutInfo.ref) - report.lfs_ms = Date.now() - lfsStart - core.endGroup() - } - - // Sparse checkout - if (!settings.sparseCheckout) { - let gitVersion = await git.version() - // no need to disable sparse-checkout if the installed git runtime doesn't even support it. - if (gitVersion.checkMinimum(MinimumGitSparseCheckoutVersion)) { - await git.disableSparseCheckout() - } - } else { - core.startGroup('Setting up sparse checkout') - if (settings.sparseCheckoutConeMode) { - await git.sparseCheckout(settings.sparseCheckout) - } else { - await git.sparseCheckoutNonConeMode(settings.sparseCheckout) - } core.endGroup() - } - - // Checkout - core.startGroup('Checking out the ref') - const checkoutStart = Date.now() - await git.checkout(checkoutInfo.ref, checkoutInfo.startPoint) - if (cacheInfo) { - // Materializing the working tree reads objects from the mirror through - // alternates — the second half of the clone-from-mirror cost. - report.clone_from_mirror_ms += Date.now() - checkoutStart } else { - // Fallback/bypass: working-tree materialization is part of the full - // checkout cost, keeping phase totals comparable across serving modes. - report.full_checkout_ms += Date.now() - checkoutStart - } - core.endGroup() - - // Dissociate from Blacksmith mirror if requested - // This copies all objects from alternates into the local repo so it's independent - if (settings.dissociate && cacheInfo) { - core.startGroup('Dissociating from Blacksmith mirror') - await blacksmithCache.dissociate( - settings.repositoryPath, - git.getEnvironment() - ) - core.endGroup() + await fetchAndCheckout(settings, git, cacheInfo, mirrorFresh, report) } // Submodules @@ -602,6 +482,274 @@ async function getSourceInner( } } +/** + * Mount the requested revision from the mirror with gitfs. Returns null, + * after logging why, when this checkout is outside the gitfs cohort; throws + * when gitfs was attempted and failed. + */ +async function mountWithGitfs( + settings: IGitSourceSettings, + cacheInfo: blacksmithCache.CacheInfo | null, + repositoryUrl: string +): Promise { + const reason = gitfs.gitfsIneligibilityReason(settings, { + mirrorPath: cacheInfo?.mirrorPath ?? null + }) + if (reason || !cacheInfo) { + core.info(`[gitfs] not used: ${reason}`) + return null + } + const binary = await gitfs.findBinary() + if (!binary) { + core.info('[gitfs] not used: gitfs binary not found on PATH') + return null + } + + const candidates = gitfs.mirrorRevisionCandidates( + settings.ref, + settings.commit + ) + const revision = await gitfs.resolveInMirror(cacheInfo.mirrorPath, candidates) + const mountStart = Date.now() + if (revision) { + await gitfs.mount( + binary, + cacheInfo.mirrorPath, + revision.sha, + settings.repositoryPath + ) + stateHelper.setGitfsMountPoint(settings.repositoryPath) + core.info( + `[gitfs] ${settings.repositoryPath} serves ${revision.sha} from the mirror (mounted in ${Date.now() - mountStart} ms); the tree is written out in the background` + ) + return revision + } + // The mirror lacks the revision (an empty mirror, a push it has not + // synced, a pull-request merge ref): gitfs fetches it from the remote into + // the mirror and serves the pack as it arrives; the rest of the repository + // follows in the background. The mirror changes, so the post step commits + // it. + const rev = gitfs.coldRevision(settings.ref, settings.commit) + core.info(`[gitfs] ${rev} is not in the mirror, streaming it from the remote`) + const sha = await gitfs.mountCold( + binary, + cacheInfo.mirrorPath, + rev, + settings.repositoryPath, + repositoryUrl, + settings.authToken + ) + stateHelper.setGitfsMountPoint(settings.repositoryPath) + stateHelper.setBlacksmithCacheMirrorChanged(true) + const {mirrorRef, kind} = candidates[0] + core.info( + `[gitfs] ${settings.repositoryPath} serves ${sha} streamed from the remote (mounted in ${Date.now() - mountStart} ms); the mirror is completed in the background` + ) + return {sha, mirrorRef, kind} +} + +/** + * The regular path: fetch the requested revision into the workspace (from + * the mirror where possible) and check it out. + */ +async function fetchAndCheckout( + settings: IGitSourceSettings, + git: IGitCommandManager, + cacheInfo: blacksmithCache.CacheInfo | null, + mirrorFresh: boolean, + report: mirrorTelemetry.CheckoutReport +): Promise { + // Fetch + core.startGroup('Fetching the repository') + const objectsDir = path.join(settings.repositoryPath, '.git', 'objects') + const objectsBytesBefore = cacheInfo + ? await mirrorTelemetry.dirSizeBytesOrNull(objectsDir) + : null + const fetchStart = Date.now() + const fetchOptions: FetchOptions = {} + + if (settings.filter) { + fetchOptions.filter = settings.filter + } else if (settings.sparseCheckout) { + fetchOptions.filter = 'blob:none' + } + + if (settings.fetchDepth <= 0) { + // When the Blacksmith mirror is available and synced with the remote, + // copy branch/tag refs from it locally (objects are already shared via + // alternates) instead of doing a full +refs/heads/* fetch over the + // network, and only ask the network for the specific ref/commit this + // run needs. + let fetchedFromMirror = false + if ( + cacheInfo && + mirrorFresh && + !fetchOptions.filter && + !fsHelper.fileExistsSync( + path.join(settings.repositoryPath, '.git', 'shallow') + ) + ) { + fetchedFromMirror = await blacksmithCache.fetchRefsFromMirror( + settings.repositoryPath, + cacheInfo.mirrorPath, + git.getEnvironment() + ) + } + + if (fetchedFromMirror) { + // The mirror copy only materializes branches and tags. Any other ref + // (refs/pull/*, etc.) must be fetched from the network so its local + // destination ref exists for checkout. For branches/tags, verify the + // wanted ref/commit is present and fetch it directly if not. + const upperRef = (settings.ref || '').toUpperCase() + const coveredByMirror = + !settings.ref || + !upperRef.startsWith('REFS/') || + upperRef.startsWith('REFS/HEADS/') || + upperRef.startsWith('REFS/TAGS/') + let verified = false + if (coveredByMirror) { + verified = await refHelper.testRef(git, settings.ref, settings.commit) + if (verified && settings.commit) { + verified = await git.shaExists(settings.commit) + } + } + if (!verified) { + const refSpec = refHelper.getRefSpec(settings.ref, settings.commit) + await git.fetch(refSpec, fetchOptions) + } + } else { + // Fetch all branches and tags + let refSpec = refHelper.getRefSpecForAllHistory( + settings.ref, + settings.commit + ) + await git.fetch(refSpec, fetchOptions) + + // When all history is fetched, the ref we're interested in may have moved to a different + // commit (push or force push). If so, fetch again with a targeted refspec. + if (!(await refHelper.testRef(git, settings.ref, settings.commit))) { + refSpec = refHelper.getRefSpec(settings.ref, settings.commit) + await git.fetch(refSpec, fetchOptions) + } + } + } else { + fetchOptions.fetchDepth = settings.fetchDepth + fetchOptions.fetchTags = settings.fetchTags + // With the mirror attached as an alternate, git would treat every + // mirror ref as a known tip: the deepening fetch's connectivity check + // walks the whole mirror object graph and negotiation offers every + // mirror ref. Turn alternate ref discovery off and offer a few mirror + // tips explicitly instead, so the server still sends only the delta + // from the mirror (see resolveShallowNegotiationTips). + if ( + cacheInfo && + (await git.version()).checkMinimum( + MinimumGitAlternateRefsCommandVersion + ) && + (await blacksmithCache.sharesMirrorObjects( + settings.repositoryPath, + cacheInfo.mirrorPath + )) + ) { + fetchOptions.ignoreAlternateRefs = true + fetchOptions.negotiationTips = + await blacksmithCache.resolveShallowNegotiationTips( + cacheInfo.mirrorPath, + settings.ref, + process.env['GITHUB_BASE_REF'] || '' + ) + core.info( + `[git-mirror] Shallow fetch negotiating from ${fetchOptions.negotiationTips.length} mirror tip(s) with alternate ref discovery disabled` + ) + } + const refSpec = refHelper.getRefSpec(settings.ref, settings.commit) + await git.fetch(refSpec, fetchOptions) + } + if (cacheInfo) { + // Mirror-served: the fetch pulls only the delta the mirror is missing. + report.delta_fetch_ms = Date.now() - fetchStart + const objectsBytesAfter = + await mirrorTelemetry.dirSizeBytesOrNull(objectsDir) + report.delta_fetch_bytes = + objectsBytesBefore !== null && objectsBytesAfter !== null + ? Math.max(0, objectsBytesAfter - objectsBytesBefore) + : 0 + report.mirror_size_bytes = await mirrorTelemetry.packSizeBytes( + cacheInfo.mirrorPath + ) + report.ref_count = await mirrorTelemetry.refCount(cacheInfo.mirrorPath) + } else { + // Fallback/bypass: the fetch is the full checkout cost. + report.full_checkout_ms = Date.now() - fetchStart + } + core.endGroup() + + // Checkout info + core.startGroup('Determining the checkout info') + const checkoutInfo = await refHelper.getCheckoutInfo( + git, + settings.ref, + settings.commit + ) + core.endGroup() + + // LFS fetch + // Explicit lfs-fetch to avoid slow checkout (fetches one lfs object at a time). + // Explicit lfs fetch will fetch lfs objects in parallel. + // For sparse checkouts, let `checkout` fetch the needed objects lazily. + if (settings.lfs && !settings.sparseCheckout) { + core.startGroup('Fetching LFS objects') + const lfsStart = Date.now() + await git.lfsFetch(checkoutInfo.startPoint || checkoutInfo.ref) + report.lfs_ms = Date.now() - lfsStart + core.endGroup() + } + + // Sparse checkout + if (!settings.sparseCheckout) { + let gitVersion = await git.version() + // no need to disable sparse-checkout if the installed git runtime doesn't even support it. + if (gitVersion.checkMinimum(MinimumGitSparseCheckoutVersion)) { + await git.disableSparseCheckout() + } + } else { + core.startGroup('Setting up sparse checkout') + if (settings.sparseCheckoutConeMode) { + await git.sparseCheckout(settings.sparseCheckout) + } else { + await git.sparseCheckoutNonConeMode(settings.sparseCheckout) + } + core.endGroup() + } + + // Checkout + core.startGroup('Checking out the ref') + const checkoutStart = Date.now() + await git.checkout(checkoutInfo.ref, checkoutInfo.startPoint) + if (cacheInfo) { + // Materializing the working tree reads objects from the mirror through + // alternates — the second half of the clone-from-mirror cost. + report.clone_from_mirror_ms += Date.now() - checkoutStart + } else { + // Fallback/bypass: working-tree materialization is part of the full + // checkout cost, keeping phase totals comparable across serving modes. + report.full_checkout_ms += Date.now() - checkoutStart + } + core.endGroup() + + // Dissociate from Blacksmith mirror if requested + // This copies all objects from alternates into the local repo so it's independent + if (settings.dissociate && cacheInfo) { + core.startGroup('Dissociating from Blacksmith mirror') + await blacksmithCache.dissociate( + settings.repositoryPath, + git.getEnvironment() + ) + core.endGroup() + } +} + export async function cleanup(repositoryPath: string): Promise { // Repo exists? if ( diff --git a/src/gitfs.ts b/src/gitfs.ts new file mode 100644 index 0000000..656d26e --- /dev/null +++ b/src/gitfs.ts @@ -0,0 +1,391 @@ +import * as core from '@actions/core' +import * as exec from '@actions/exec' +import * as fs from 'fs' +import * as io from '@actions/io' +import * as path from 'path' +import {isRunningInContainer} from './container-detector' +import {getAuthConfigArgs} from './blacksmith-cache' + +/** + * gitfs serves the workspace from the git mirror sticky disk instead of + * checking it out: `gitfs mount --ref --daemon` + * returns as soon as the tree is mounted (milliseconds), writes the real + * checkout out in the background and swaps it in for the mount once it is + * complete. Writes made through the mount in the meantime (`.git`, npm's + * node_modules, ...) land in that same tree. The agent enables it per job via + * BLACKSMITH_USE_GITFS=true, driven by the installation's use_gitfs_checkout + * flag; anything gitfs cannot serve falls through to the regular checkout. + */ +export const GITFS_ENV = 'BLACKSMITH_USE_GITFS' +const GITFS_BINARY = 'gitfs' +const DAEMON_EXIT_TIMEOUT_MS = 10_000 +const DAEMON_KILL_TIMEOUT_MS = 5_000 + +export function isGitfsRequested( + env: {[key: string]: string | undefined} = process.env +): boolean { + return env[GITFS_ENV] === 'true' +} + +export interface EligibilitySettings { + ref: string + commit: string + submodules: boolean + lfs: boolean + sparseCheckout: string[] | null | undefined + filter: string | undefined + dissociate: boolean + repositoryPath: string +} + +export interface EligibilityContext { + mirrorPath: string | null + platform?: string + inContainer?: boolean +} + +/** + * Why gitfs cannot serve this checkout, or null when it can. The v1 cohort is + * the plain case: a Linux runner VM with the mirror mounted, a branch, tag, + * pull ref or commit to serve, and none of the features that need the + * workspace to be a real fetch (submodules, LFS, sparse/partial clones, + * dissociate). + */ +export function gitfsIneligibilityReason( + settings: EligibilitySettings, + ctx: EligibilityContext +): string | null { + const platform = ctx.platform ?? process.platform + if (platform !== 'linux') { + return `platform ${platform} is not linux` + } + if (!ctx.mirrorPath) { + return 'git mirror is not available' + } + const inContainer = ctx.inContainer ?? isRunningInContainer() + if (inContainer) { + return 'running inside a container' + } + if (!settings.ref && !settings.commit) { + return 'no ref or commit requested' + } + if (settings.submodules) { + return 'submodules are enabled' + } + if (settings.lfs) { + return 'lfs is enabled' + } + if (settings.sparseCheckout && settings.sparseCheckout.length > 0) { + return 'sparse checkout is enabled' + } + if (settings.filter) { + return `filter ${settings.filter} is set` + } + if (settings.dissociate) { + return 'dissociate is enabled' + } + if (fs.existsSync(path.join(settings.repositoryPath, '.git'))) { + return 'workspace already contains a repository' + } + return null +} + +export async function findBinary(): Promise { + const found = await io.which(GITFS_BINARY, false) + return found || null +} + +export type RefKind = 'branch' | 'tag' | 'pull' | 'other' | 'commit' + +export interface MirrorRevision { + sha: string + /** The mirror ref the revision came from, '' when only a commit matched. */ + mirrorRef: string + kind: RefKind +} + +interface Candidate { + rev: string + mirrorRef: string + kind: RefKind +} + +function classify(ref: string): RefKind { + const upper = ref.toUpperCase() + if (upper.startsWith('REFS/HEADS/')) return 'branch' + if (upper.startsWith('REFS/TAGS/')) return 'tag' + if (upper.startsWith('REFS/PULL/')) return 'pull' + return 'other' +} + +/** + * Revisions to try in the mirror for the requested ref/commit, in order. A + * commit wins over the ref (that is what the checkout would end up at); an + * unqualified ref is a branch first, then a tag, as in getCheckoutInfo. + */ +export function mirrorRevisionCandidates( + ref: string, + commit: string +): Candidate[] { + if (commit) { + const kind = ref ? classify(ref) : 'commit' + const mirrorRef = ref && ref.toUpperCase().startsWith('REFS/') ? ref : '' + return [{rev: `${commit}^{commit}`, mirrorRef, kind}] + } + if (!ref) { + return [] + } + if (ref.toUpperCase().startsWith('REFS/')) { + return [{rev: `${ref}^{commit}`, mirrorRef: ref, kind: classify(ref)}] + } + return [ + { + rev: `refs/heads/${ref}^{commit}`, + mirrorRef: `refs/heads/${ref}`, + kind: 'branch' + }, + { + rev: `refs/tags/${ref}^{commit}`, + mirrorRef: `refs/tags/${ref}`, + kind: 'tag' + } + ] +} + +/** + * What `gitfs mount --url` fetches for a revision the mirror lacks: a commit + * by sha, a qualified ref by name, an unqualified ref as a branch. + */ +export function coldRevision(ref: string, commit: string): string { + if (commit) { + return commit + } + if (ref.toUpperCase().startsWith('REFS/')) { + return ref + } + return `refs/heads/${ref}` +} + +export async function resolveInMirror( + mirrorPath: string, + candidates: Candidate[] +): Promise { + for (const candidate of candidates) { + const result = await exec.getExecOutput( + 'git', + ['-C', mirrorPath, 'rev-parse', '--verify', '--quiet', candidate.rev], + {silent: true, ignoreReturnCode: true} + ) + const sha = result.stdout.trim() + if (result.exitCode === 0 && /^[0-9a-f]{40}$/.test(sha)) { + return {sha, mirrorRef: candidate.mirrorRef, kind: candidate.kind} + } + } + return null +} + +export async function fetchIntoMirror( + mirrorPath: string, + repositoryUrl: string, + authToken: string, + refspecs: string[] +): Promise { + const auth = getAuthConfigArgs(repositoryUrl, authToken) + await exec.exec('git', [ + '-C', + mirrorPath, + '-c', + `${auth.configKey}=${auth.configValue}`, + '-c', + 'gc.auto=0', + 'fetch', + '--no-tags', + '--no-recurse-submodules', + '--no-write-fetch-head', + repositoryUrl, + ...refspecs + ]) +} + +export async function mount( + binary: string, + mirrorPath: string, + sha: string, + mountpoint: string +): Promise { + if (fs.existsSync(mountpoint) && fs.readdirSync(mountpoint).length > 0) { + throw new Error(`${mountpoint} is not empty`) + } + await exec.exec(binary, [ + 'mount', + mirrorPath, + '--ref', + sha, + mountpoint, + '--daemon' + ]) +} + +/** + * Mount a revision the mirror does not have: gitfs fetches it from the + * remote into the mirror (`--depth 1`, serving the pack as it arrives) and + * completes the mirror in the background. Returns the commit sha, which + * gitfs prints once the commit is in, before git has written the ref. + */ +export async function mountCold( + binary: string, + mirrorPath: string, + rev: string, + mountpoint: string, + repositoryUrl: string, + authToken: string +): Promise { + if (fs.existsSync(mountpoint) && fs.readdirSync(mountpoint).length > 0) { + throw new Error(`${mountpoint} is not empty`) + } + const env: {[key: string]: string} = {} + for (const [key, value] of Object.entries(process.env)) { + if (value !== undefined) { + env[key] = value + } + } + env.GITFS_TOKEN = authToken + const result = await exec.getExecOutput( + binary, + [ + 'mount', + mirrorPath, + '--ref', + rev, + mountpoint, + '--daemon', + '--url', + repositoryUrl + ], + {env} + ) + const sha = result.stdout.trim().split('\n').pop()?.trim() ?? '' + if (!/^[0-9a-f]{40}$/.test(sha)) { + throw new Error( + `gitfs did not report the mounted commit: ${result.stdout.trim()}` + ) + } + return sha +} + +/** + * Point the workspace at the mounted revision without touching the tree: + * the local branch / tag / pull ref the regular checkout would have created, + * HEAD on it (or detached, as `git checkout` would leave it), and an index + * for the tree. The files are already there, served by gitfs. + */ +export async function pointWorkspaceAt( + repositoryPath: string, + ref: string, + revision: MirrorRevision, + env?: {[key: string]: string} +): Promise { + const git = async (...args: string[]): Promise => { + await exec.exec('git', ['-C', repositoryPath, ...args], {env}) + } + const {sha} = revision + if (revision.kind === 'branch') { + const branch = revision.mirrorRef.slice('refs/heads/'.length) + await git('update-ref', `refs/remotes/origin/${branch}`, sha) + await git('update-ref', `refs/heads/${branch}`, sha) + await git('config', `branch.${branch}.remote`, 'origin') + await git('config', `branch.${branch}.merge`, `refs/heads/${branch}`) + await git('symbolic-ref', 'HEAD', `refs/heads/${branch}`) + } else { + if (revision.kind === 'tag') { + await git('update-ref', revision.mirrorRef, sha) + } else if (revision.kind === 'pull') { + const name = revision.mirrorRef.slice('refs/pull/'.length) + await git('update-ref', `refs/remotes/pull/${name}`, sha) + } + await git('update-ref', '--no-deref', 'HEAD', sha) + } + // Index entries without stat data: git re-verifies file contents on the + // first status/diff instead of touching every file now. + await git('read-tree', sha) +} + +function daemonPids(mirrorPath: string): number[] { + const pids: number[] = [] + let entries: string[] + try { + entries = fs.readdirSync('/proc') + } catch { + return pids + } + for (const entry of entries) { + if (!/^\d+$/.test(entry)) continue + let cmdline: string + try { + cmdline = fs.readFileSync(`/proc/${entry}/cmdline`, 'utf8') + } catch { + continue + } + const argv = cmdline.split('\0') + if ( + path.basename(argv[0] || '') === GITFS_BINARY && + argv.includes('mount') && + argv.includes(mirrorPath) + ) { + pids.push(Number(entry)) + } + } + return pids +} + +async function waitForDaemons( + mirrorPath: string, + timeoutMs: number +): Promise { + const deadline = Date.now() + timeoutMs + let pids = daemonPids(mirrorPath) + while (pids.length > 0 && Date.now() < deadline) { + await new Promise(resolve => setTimeout(resolve, 100)) + pids = daemonPids(mirrorPath) + } + return pids +} + +/** + * Take gitfs down before the mirror sticky disk is unmounted: the daemon has + * the mirror's packs mapped, so the disk cannot be released while it lives. + * `gitfs unmount` handles both phases — while the tree is still being + * written the workspace path is a symlink into the mount and unmounting + * discards the partial tree (the workspace is left empty); after the swap + * the path is a plain directory and the FUSE view that served it is still up + * on the hidden sibling path, which unmount takes down. + */ +export async function teardown( + mountpoint: string, + mirrorPath: string +): Promise { + const binary = await findBinary() + if (binary && fs.existsSync(mountpoint)) { + core.info(`[gitfs] Unmounting ${mountpoint}`) + await exec.exec(binary, ['unmount', mountpoint], {ignoreReturnCode: true}) + } + let pids = await waitForDaemons(mirrorPath, DAEMON_EXIT_TIMEOUT_MS) + if (pids.length > 0) { + core.info( + `[gitfs] Stopping daemon(s) ${pids.join(', ')} still serving ${mirrorPath}` + ) + for (const pid of pids) { + try { + process.kill(pid, 'SIGTERM') + } catch { + // already gone + } + } + pids = await waitForDaemons(mirrorPath, DAEMON_KILL_TIMEOUT_MS) + if (pids.length > 0) { + core.warning( + `[gitfs] daemon(s) ${pids.join(', ')} did not exit; the mirror unmount may fail` + ) + } + } +} diff --git a/src/main.ts b/src/main.ts index f875dc9..80c1f6b 100644 --- a/src/main.ts +++ b/src/main.ts @@ -5,6 +5,7 @@ import * as inputHelper from './input-helper' import * as path from 'path' import * as stateHelper from './state-helper' import * as blacksmithCache from './blacksmith-cache' +import * as gitfs from './gitfs' import {checkPreviousStepFailures} from './step-checker' import {reportInternalMetric} from './internal-metrics' import * as mirrorTelemetry from './mirror-telemetry' @@ -46,6 +47,16 @@ async function cleanup(): Promise { const repoName = stateHelper.BlacksmithCacheRepoName const mountPoint = stateHelper.BlacksmithCacheMountPoint const mirrorPath = stateHelper.BlacksmithCacheMirrorPath + + // gitfs maps the mirror's packs, so it has to be gone before the sticky + // disk can be unmounted below. + if (stateHelper.GitfsMountPoint && mirrorPath) { + try { + await gitfs.teardown(stateHelper.GitfsMountPoint, mirrorPath) + } catch (error) { + core.warning(`[gitfs] teardown failed: ${error}`) + } + } const performedHydration = stateHelper.BlacksmithCachePerformedHydration let mirrorChanged = stateHelper.BlacksmithCacheMirrorChanged let mirrorSyncFailed = stateHelper.BlacksmithCacheMirrorSyncFailed diff --git a/src/state-helper.ts b/src/state-helper.ts index e420115..f1158f3 100644 --- a/src/state-helper.ts +++ b/src/state-helper.ts @@ -102,6 +102,12 @@ export const BlacksmithCacheVerbose = * Reason the host gave at mount time for denying this job's sticky disk commit * (e.g. branch protection). Empty when no denial was reported. */ +/** + * Workspace path served by gitfs, so the POST action can take the mount down + * before the mirror sticky disk is unmounted. + */ +export const GitfsMountPoint = core.getState('gitfsMountPoint') + export const BlacksmithCacheCommitEarlyDenyReason = core.getState( 'blacksmithCacheCommitEarlyDenyReason' ) @@ -233,6 +239,10 @@ export function setBlacksmithCacheCommitEarlyDenyReason(reason: string) { core.saveState('blacksmithCacheCommitEarlyDenyReason', reason) } +export function setGitfsMountPoint(mountPoint: string): void { + core.saveState('gitfsMountPoint', mountPoint) +} + // Publish a variable so that when the POST action runs, it can determine it should run the cleanup logic. // This is necessary since we don't have a separate entry point. if (!IsPost) {