diff --git a/.licenses/npm/@buf/blacksmith_vm-agent.bufbuild_es-1.10.0-20260904133914-c6beb298c70c.1.dep.yml b/.licenses/npm/@buf/blacksmith_vm-agent.bufbuild_es-1.10.0-20261023032817-36eedc7ae378.1.dep.yml similarity index 83% rename from .licenses/npm/@buf/blacksmith_vm-agent.bufbuild_es-1.10.0-20260904133914-c6beb298c70c.1.dep.yml rename to .licenses/npm/@buf/blacksmith_vm-agent.bufbuild_es-1.10.0-20261023032817-36eedc7ae378.1.dep.yml index 888c671..5a70426 100644 --- a/.licenses/npm/@buf/blacksmith_vm-agent.bufbuild_es-1.10.0-20260904133914-c6beb298c70c.1.dep.yml +++ b/.licenses/npm/@buf/blacksmith_vm-agent.bufbuild_es-1.10.0-20261023032817-36eedc7ae378.1.dep.yml @@ -1,8 +1,8 @@ --- name: "@buf/blacksmith_vm-agent.bufbuild_es" -version: 1.10.0-20260904133914-c6beb298c70c.1 +version: 1.10.0-20261023032817-36eedc7ae378.1 type: npm -summary: This package contains generated code for the protobuf module buf.build/blacksmith/vm-agent:c6beb298c70c4bfcb38cf72e37c67c91 +summary: This package contains generated code for the protobuf module buf.build/blacksmith/vm-agent:36eedc7ae3784f628eaef54d72088d60 when generated with the plugin buf.build/bufbuild/es:v1.10.0 revision:1 homepage: license: apache-2.0 diff --git a/.licenses/npm/@buf/blacksmith_vm-agent.bufbuild_es-1.9.0-20260904133914-c6beb298c70c.2.dep.yml b/.licenses/npm/@buf/blacksmith_vm-agent.bufbuild_es-1.9.0-20261023032817-36eedc7ae378.2.dep.yml similarity index 83% rename from .licenses/npm/@buf/blacksmith_vm-agent.bufbuild_es-1.9.0-20260904133914-c6beb298c70c.2.dep.yml rename to .licenses/npm/@buf/blacksmith_vm-agent.bufbuild_es-1.9.0-20261023032817-36eedc7ae378.2.dep.yml index b491026..8b16c1e 100644 --- a/.licenses/npm/@buf/blacksmith_vm-agent.bufbuild_es-1.9.0-20260904133914-c6beb298c70c.2.dep.yml +++ b/.licenses/npm/@buf/blacksmith_vm-agent.bufbuild_es-1.9.0-20261023032817-36eedc7ae378.2.dep.yml @@ -1,8 +1,8 @@ --- name: "@buf/blacksmith_vm-agent.bufbuild_es" -version: 1.9.0-20260904133914-c6beb298c70c.2 +version: 1.9.0-20261023032817-36eedc7ae378.2 type: npm -summary: This package contains generated code for the protobuf module buf.build/blacksmith/vm-agent:c6beb298c70c4bfcb38cf72e37c67c91 +summary: This package contains generated code for the protobuf module buf.build/blacksmith/vm-agent:36eedc7ae3784f628eaef54d72088d60 when generated with the plugin buf.build/bufbuild/es:v1.9.0 revision:2 homepage: license: apache-2.0 diff --git a/.licenses/npm/@buf/blacksmith_vm-agent.connectrpc_es.dep.yml b/.licenses/npm/@buf/blacksmith_vm-agent.connectrpc_es.dep.yml index 8dbbec0..a095e0f 100644 --- a/.licenses/npm/@buf/blacksmith_vm-agent.connectrpc_es.dep.yml +++ b/.licenses/npm/@buf/blacksmith_vm-agent.connectrpc_es.dep.yml @@ -1,8 +1,8 @@ --- name: "@buf/blacksmith_vm-agent.connectrpc_es" -version: 1.6.1-20260904133914-c6beb298c70c.2 +version: 1.6.1-20261023032817-36eedc7ae378.2 type: npm -summary: This package contains generated code for the protobuf module buf.build/blacksmith/vm-agent:c6beb298c70c4bfcb38cf72e37c67c91 +summary: This package contains generated code for the protobuf module buf.build/blacksmith/vm-agent:36eedc7ae3784f628eaef54d72088d60 when generated with the plugin buf.build/connectrpc/es:v1.6.1 revision:2 homepage: license: apache-2.0 diff --git a/__test__/blacksmith-cache.test.ts b/__test__/blacksmith-cache.test.ts index 5f50057..acc9b03 100644 --- a/__test__/blacksmith-cache.test.ts +++ b/__test__/blacksmith-cache.test.ts @@ -21,6 +21,33 @@ jest.mock('../src/container-detector', () => ({ })) import * as blacksmithCache from '../src/blacksmith-cache' + +describe('parseFstrimMinExtentBytes', () => { + it('accepts the RPC bigint or saved-state string and maps junk to 0', () => { + expect(blacksmithCache.parseFstrimMinExtentBytes(BigInt(65536))).toBe(65536) + expect(blacksmithCache.parseFstrimMinExtentBytes('4194304')).toBe(4194304) + expect(blacksmithCache.parseFstrimMinExtentBytes(BigInt(0))).toBe(0) + expect(blacksmithCache.parseFstrimMinExtentBytes(-1)).toBe(0) + expect(blacksmithCache.parseFstrimMinExtentBytes(undefined)).toBe(0) + }) +}) + +describe('parseFstrimTimeoutSecs', () => { + it('accepts the RPC bigint, a number, or the saved-state string', () => { + expect(blacksmithCache.parseFstrimTimeoutSecs(BigInt(5))).toBe(5) + expect(blacksmithCache.parseFstrimTimeoutSecs(5)).toBe(5) + expect(blacksmithCache.parseFstrimTimeoutSecs('5')).toBe(5) + expect(blacksmithCache.parseFstrimTimeoutSecs('2.9')).toBe(2) + }) + + it('treats anything unusable as disabled', () => { + expect(blacksmithCache.parseFstrimTimeoutSecs(BigInt(0))).toBe(0) + expect(blacksmithCache.parseFstrimTimeoutSecs(-1)).toBe(0) + expect(blacksmithCache.parseFstrimTimeoutSecs('')).toBe(0) + expect(blacksmithCache.parseFstrimTimeoutSecs(undefined)).toBe(0) + expect(blacksmithCache.parseFstrimTimeoutSecs('nope')).toBe(0) + }) +}) import {isRunningInContainer} from '../src/container-detector' const mockIsRunningInContainer = isRunningInContainer as jest.MockedFunction< diff --git a/dist/index.js b/dist/index.js index d1d1519..0d031c6 100644 --- a/dist/index.js +++ b/dist/index.js @@ -69,6 +69,8 @@ exports.dissociate = dissociate; exports.hasCommitGraph = hasCommitGraph; exports.markKeepPacks = markKeepPacks; exports.runMirrorMaintenance = runMirrorMaintenance; +exports.parseFstrimTimeoutSecs = parseFstrimTimeoutSecs; +exports.parseFstrimMinExtentBytes = parseFstrimMinExtentBytes; exports.cleanup = cleanup; const core = __importStar(__nccwpck_require__(2186)); const exec = __importStar(__nccwpck_require__(1514)); @@ -344,7 +346,9 @@ function setupCache(owner, repo) { hydrationInProgress: true, hydrationMessage, performedHydration: false, - commitEarlyDenyReason: '' + commitEarlyDenyReason: '', + fstrimTimeoutSecs: 0, + fstrimMinExtentBytes: 0 }; } // Re-throw other errors @@ -365,6 +369,8 @@ function setupCache(owner, repo) { const commitEarlyDenyReason = response.commitEarlyDeny ? response.commitEarlyDenyReason || 'denied by host policy' : ''; + const fstrimTimeoutSecs = parseFstrimTimeoutSecs(response.fstrimTimeoutSecs); + const fstrimMinExtentBytes = parseFstrimMinExtentBytes(response.fstrimMinExtentBytes); if (commitEarlyDenyReason) { core.notice(`[git-mirror] Sticky disk changes will not be committed for this job (${commitEarlyDenyReason}). The git mirror cache is used as-is and any changes to it are discarded.`); } @@ -387,7 +393,9 @@ function setupCache(owner, repo) { mirrorPath: getMirrorPath(owner, repo), hydrationInProgress: false, performedHydration: false, // Will be set by ensureMirror if we do initial clone - commitEarlyDenyReason + commitEarlyDenyReason, + fstrimTimeoutSecs, + fstrimMinExtentBytes }; }); } @@ -1936,17 +1944,72 @@ function flushBlockDevice(devicePath) { } }); } +/** + * Parse the host's fstrim budget (GetStickyDiskResponse.fstrim_timeout_secs, + * a protobuf int64 that arrives as a bigint, or the string read back from + * action state) into whole seconds. Anything unusable is 0, which disables + * the trim. + */ +function parseFstrimTimeoutSecs(value) { + const n = typeof value === 'bigint' ? Number(value) : Number(value !== null && value !== void 0 ? value : 0); + if (!Number.isFinite(n) || n <= 0) + return 0; + return Math.floor(n); +} +/** + * Parse the host's `fstrim --minimum` value + * (GetStickyDiskResponse.fstrim_min_extent_bytes, a bigint or saved-state + * string) into whole bytes. Anything unusable is 0: the fstrim default. + */ +function parseFstrimMinExtentBytes(value) { + return parseFstrimTimeoutSecs(value); +} +/** + * Best-effort `fstrim` of the mirror filesystem before it is unmounted and + * committed: deleted packs keep their RBD extents allocated on the backing + * store until the guest discards them. Bounded by the host-supplied budget; + * any failure (discard not negotiated, missing binary, timeout) only logs. + */ +function trimFilesystem(mountPoint, timeoutSecs, minExtentBytes) { + return __awaiter(this, void 0, void 0, function* () { + const start = Date.now(); + try { + const result = yield exec.getExecOutput('timeout', [ + String(timeoutSecs), + 'sudo', + 'fstrim', + '-v', + ...(minExtentBytes > 0 ? ['-m', String(minExtentBytes)] : []), + '--', + mountPoint + ], { ignoreReturnCode: true, silent: true }); + const durationMs = Date.now() - start; + if (result.exitCode === 0) { + core.info(`[git-mirror] fstrim ${mountPoint}: ${result.stdout.trim() || 'done'} in ${durationMs}ms`); + } + else if (result.exitCode === TIMEOUT_EXIT_CODE) { + core.info(`[git-mirror] fstrim ${mountPoint}: timed out after ${timeoutSecs}s`); + } + else { + core.info(`[git-mirror] fstrim ${mountPoint}: failed in ${durationMs}ms: ${result.stderr.trim()}`); + } + } + catch (error) { + core.info(`[git-mirror] fstrim ${mountPoint}: failed: ${error.message}`); + } + }); +} /** * Cleanup: run mirror maintenance, sync, unmount, and commit the sticky disk. * - * Execution order: maintenance → sync → unmount (with retry) → flush → commit + * Execution order: maintenance → sync → fstrim → unmount (with retry) → flush → commit * If the mirror sync failed or timed out, shouldCommit is set to false. * Maintenance failure or timeout does not affect the commit. */ function cleanup(options) { return __awaiter(this, void 0, void 0, function* () { var _a; - const { exposeId, stickyDiskKey, repoName, mountPoint, mirrorPath, mirrorSyncFailed, mirrorSyncTimedOut, vmHydratedGitMirror } = options; + const { exposeId, stickyDiskKey, repoName, mountPoint, mirrorPath, mirrorSyncFailed, mirrorSyncTimedOut, vmHydratedGitMirror, fstrimTimeoutSecs, fstrimMinExtentBytes } = options; let { shouldCommit } = options; const result = { // skipped until maintenance actually runs, so a mirror that is not @@ -1980,6 +2043,15 @@ function cleanup(options) { catch (_b) { core.warning('[git-mirror] Failed to sync filesystem'); } + // Discard freed blocks before unmount so the committed snapshot does not + // keep them allocated. Only worth it when the result is persisted, and only + // when the host has enabled it. + if (mountPoint && + shouldCommit && + fstrimTimeoutSecs && + fstrimTimeoutSecs > 0) { + yield trimFilesystem(mountPoint, fstrimTimeoutSecs, fstrimMinExtentBytes !== null && fstrimMinExtentBytes !== void 0 ? fstrimMinExtentBytes : 0); + } // Get device path before unmount for durability flush let devicePath = null; if (mountPoint) { @@ -3727,6 +3799,8 @@ function getSourceInner(settings, report, onHydrationReport) { stateHelper.setBlacksmithCacheMirrorPath(cacheInfo.mirrorPath); stateHelper.setBlacksmithCacheMountPoint(cacheInfo.mountPoint); stateHelper.setBlacksmithCacheCommitEarlyDenyReason(cacheInfo.commitEarlyDenyReason); + stateHelper.setBlacksmithCacheFstrimTimeoutSecs(cacheInfo.fstrimTimeoutSecs); + stateHelper.setBlacksmithCacheFstrimMinExtentBytes(cacheInfo.fstrimMinExtentBytes); const mirrorExisted = fsHelper.directoryExistsSync(cacheInfo.mirrorPath); const cloneStart = Date.now(); let performedHydration = false; @@ -4820,7 +4894,9 @@ function cleanup() { shouldCommit, vmHydratedGitMirror: performedHydration, mirrorSyncFailed, - mirrorSyncTimedOut + mirrorSyncTimedOut, + fstrimTimeoutSecs: stateHelper.BlacksmithCacheFstrimTimeoutSecs, + fstrimMinExtentBytes: stateHelper.BlacksmithCacheFstrimMinExtentBytes }); } catch (error) { @@ -5524,7 +5600,7 @@ var __importStar = (this && this.__importStar) || function (mod) { return result; }; Object.defineProperty(exports, "__esModule", ({ value: true })); -exports.BlacksmithCacheCommitEarlyDenyReason = exports.BlacksmithCacheVerbose = exports.BlacksmithCacheRepoUrl = exports.BlacksmithCacheMirrorSyncDeferred = exports.BlacksmithCacheMirrorSyncTimedOut = exports.BlacksmithCacheMirrorSyncFailed = exports.BlacksmithCacheMirrorChanged = exports.BlacksmithCachePerformedHydration = exports.BlacksmithCacheStickyDiskKey = exports.BlacksmithCacheRepoName = exports.BlacksmithCacheMountPoint = exports.BlacksmithCacheMirrorPath = exports.BlacksmithCacheExposeId = exports.SshKnownHostsPath = exports.SshKeyPath = exports.PostSetSafeDirectory = exports.RepositoryPath = exports.IsPost = void 0; +exports.BlacksmithCacheFstrimMinExtentBytes = exports.BlacksmithCacheFstrimTimeoutSecs = exports.BlacksmithCacheCommitEarlyDenyReason = exports.BlacksmithCacheVerbose = exports.BlacksmithCacheRepoUrl = exports.BlacksmithCacheMirrorSyncDeferred = exports.BlacksmithCacheMirrorSyncTimedOut = exports.BlacksmithCacheMirrorSyncFailed = exports.BlacksmithCacheMirrorChanged = exports.BlacksmithCachePerformedHydration = exports.BlacksmithCacheStickyDiskKey = exports.BlacksmithCacheRepoName = exports.BlacksmithCacheMountPoint = exports.BlacksmithCacheMirrorPath = exports.BlacksmithCacheExposeId = exports.SshKnownHostsPath = exports.SshKeyPath = exports.PostSetSafeDirectory = exports.RepositoryPath = exports.IsPost = void 0; exports.setRepositoryPath = setRepositoryPath; exports.setSshKeyPath = setSshKeyPath; exports.setSshKnownHostsPath = setSshKnownHostsPath; @@ -5542,6 +5618,8 @@ exports.setBlacksmithCacheMirrorSyncDeferred = setBlacksmithCacheMirrorSyncDefer exports.setBlacksmithCacheRepoUrl = setBlacksmithCacheRepoUrl; exports.setBlacksmithCacheVerbose = setBlacksmithCacheVerbose; exports.setBlacksmithCacheCommitEarlyDenyReason = setBlacksmithCacheCommitEarlyDenyReason; +exports.setBlacksmithCacheFstrimTimeoutSecs = setBlacksmithCacheFstrimTimeoutSecs; +exports.setBlacksmithCacheFstrimMinExtentBytes = setBlacksmithCacheFstrimMinExtentBytes; const core = __importStar(__nccwpck_require__(2186)); /** * Indicates whether the POST action is running @@ -5617,6 +5695,22 @@ exports.BlacksmithCacheVerbose = core.getState('blacksmithCacheVerbose') === 'tr * (e.g. branch protection). Empty when no denial was reported. */ exports.BlacksmithCacheCommitEarlyDenyReason = core.getState('blacksmithCacheCommitEarlyDenyReason'); +/** + * Host-supplied budget, in whole seconds, for trimming the mirror filesystem + * before unmount at commit. 0 when the host has not enabled the trim. + */ +exports.BlacksmithCacheFstrimTimeoutSecs = (() => { + const secs = Number(core.getState('blacksmithCacheFstrimTimeoutSecs') || '0'); + return Number.isFinite(secs) && secs > 0 ? Math.floor(secs) : 0; +})(); +/** + * The host's `fstrim -m` value in bytes, saved by the main action. 0 when + * the host left fstrim at its default minimum extent. + */ +exports.BlacksmithCacheFstrimMinExtentBytes = (() => { + const n = Number(core.getState('blacksmithCacheFstrimMinExtentBytes') || '0'); + return Number.isFinite(n) && n > 0 ? Math.floor(n) : 0; +})(); /** * Save the repository path so the POST action can retrieve the value. */ @@ -5718,6 +5812,18 @@ function setBlacksmithCacheVerbose(verbose) { function setBlacksmithCacheCommitEarlyDenyReason(reason) { core.saveState('blacksmithCacheCommitEarlyDenyReason', reason); } +/** + * Save the host's fstrim budget so the POST action can trim before unmount. + */ +function setBlacksmithCacheFstrimTimeoutSecs(secs) { + core.saveState('blacksmithCacheFstrimTimeoutSecs', String(secs)); +} +/** + * Save the host's `fstrim -m` value so the POST action can pass it along. + */ +function setBlacksmithCacheFstrimMinExtentBytes(bytes) { + core.saveState('blacksmithCacheFstrimMinExtentBytes', String(bytes)); +} // Publish a variable so that when the POST action runs, it can determine it should run the cleanup logic. // This is necessary since we don't have a separate entry point. if (!exports.IsPost) { @@ -66039,6 +66145,8 @@ const CommitSkipReason = /*@__PURE__*/ proto3.makeEnum( {no: 4, name: "COMMIT_SKIP_REASON_CLEANUP_ERROR", localName: "CLEANUP_ERROR"}, {no: 5, name: "COMMIT_SKIP_REASON_AMBIGUOUS", localName: "AMBIGUOUS"}, {no: 6, name: "COMMIT_SKIP_REASON_NO_EXPOSE", localName: "NO_EXPOSE"}, + {no: 7, name: "COMMIT_SKIP_REASON_SERVER_DECLINED", localName: "SERVER_DECLINED"}, + {no: 8, name: "COMMIT_SKIP_REASON_HOOK_SKIPPED", localName: "HOOK_SKIPPED"}, ], ); @@ -66055,6 +66163,78 @@ const IntegrityOutcome = /*@__PURE__*/ proto3.makeEnum( ], ); +/** + * ContainerCacheSetupOutcome classifies how docker container cache + * activation ended for a job. Every non-ACTIVATED outcome falls back to + * uncached docker on the ephemeral root disk; the job still runs. + * + * @generated from enum stickydisk.v1.ContainerCacheSetupOutcome + */ +const ContainerCacheSetupOutcome = /*@__PURE__*/ proto3.makeEnum( + "stickydisk.v1.ContainerCacheSetupOutcome", + [ + {no: 0, name: "CONTAINER_CACHE_SETUP_OUTCOME_UNSPECIFIED", localName: "UNSPECIFIED"}, + {no: 1, name: "CONTAINER_CACHE_SETUP_OUTCOME_ACTIVATED", localName: "ACTIVATED"}, + {no: 2, name: "CONTAINER_CACHE_SETUP_OUTCOME_DISABLED", localName: "DISABLED"}, + {no: 3, name: "CONTAINER_CACHE_SETUP_OUTCOME_METADATA_UNAVAILABLE", localName: "METADATA_UNAVAILABLE"}, + {no: 4, name: "CONTAINER_CACHE_SETUP_OUTCOME_EXPOSE_FAILED", localName: "EXPOSE_FAILED"}, + {no: 5, name: "CONTAINER_CACHE_SETUP_OUTCOME_DEVICE_SIZE_TIMEOUT", localName: "DEVICE_SIZE_TIMEOUT"}, + {no: 6, name: "CONTAINER_CACHE_SETUP_OUTCOME_MOUNT_FAILED", localName: "MOUNT_FAILED"}, + {no: 7, name: "CONTAINER_CACHE_SETUP_OUTCOME_UNUSED", localName: "UNUSED"}, + ], +); + +/** + * ContainerCacheCommitReason is the structured reason behind the job's + * commit-or-elide decision for the container cache disk. + * + * @generated from enum stickydisk.v1.ContainerCacheCommitReason + */ +const ContainerCacheCommitReason = /*@__PURE__*/ proto3.makeEnum( + "stickydisk.v1.ContainerCacheCommitReason", + [ + {no: 0, name: "CONTAINER_CACHE_COMMIT_REASON_UNSPECIFIED", localName: "UNSPECIFIED"}, + {no: 1, name: "CONTAINER_CACHE_COMMIT_REASON_HASH_CHANGED", localName: "HASH_CHANGED"}, + {no: 2, name: "CONTAINER_CACHE_COMMIT_REASON_NO_STORED_HASH", localName: "NO_STORED_HASH"}, + {no: 3, name: "CONTAINER_CACHE_COMMIT_REASON_FORCED_MAINTENANCE", localName: "FORCED_MAINTENANCE"}, + {no: 4, name: "CONTAINER_CACHE_COMMIT_REASON_ELIDE_MAINTENANCE_FAILED", localName: "ELIDE_MAINTENANCE_FAILED"}, + {no: 5, name: "CONTAINER_CACHE_COMMIT_REASON_ELIDE_HASH_MATCH", localName: "ELIDE_HASH_MATCH"}, + {no: 6, name: "CONTAINER_CACHE_COMMIT_REASON_HASH_UNAVAILABLE", localName: "HASH_UNAVAILABLE"}, + {no: 7, name: "CONTAINER_CACHE_COMMIT_REASON_ELIDE_GC_TIMEOUT", localName: "ELIDE_GC_TIMEOUT"}, + ], +); + +/** + * ContainerCacheCommitPostOutcome is the result of the guest's + * /commit-docker-container POST to the host agent. + * + * @generated from enum stickydisk.v1.ContainerCacheCommitPostOutcome + */ +const ContainerCacheCommitPostOutcome = /*@__PURE__*/ proto3.makeEnum( + "stickydisk.v1.ContainerCacheCommitPostOutcome", + [ + {no: 0, name: "CONTAINER_CACHE_COMMIT_POST_OUTCOME_UNSPECIFIED", localName: "UNSPECIFIED"}, + {no: 1, name: "CONTAINER_CACHE_COMMIT_POST_OUTCOME_OK", localName: "OK"}, + {no: 2, name: "CONTAINER_CACHE_COMMIT_POST_OUTCOME_FAILED", localName: "FAILED"}, + {no: 3, name: "CONTAINER_CACHE_COMMIT_POST_OUTCOME_SKIPPED_MISSING_IDENTITY", localName: "SKIPPED_MISSING_IDENTITY"}, + ], +); + +/** + * ContainerServeClass classifies how a container's image was served. + * + * @generated from enum stickydisk.v1.ContainerServeClass + */ +const ContainerServeClass = /*@__PURE__*/ proto3.makeEnum( + "stickydisk.v1.ContainerServeClass", + [ + {no: 0, name: "CONTAINER_SERVE_CLASS_UNSPECIFIED", localName: "UNSPECIFIED"}, + {no: 1, name: "CONTAINER_SERVE_CLASS_CACHED", localName: "CACHED"}, + {no: 2, name: "CONTAINER_SERVE_CLASS_PULLED_THIS_JOB", localName: "PULLED_THIS_JOB"}, + {no: 3, name: "CONTAINER_SERVE_CLASS_UNKNOWN", localName: "UNKNOWN"}, + ], +); + /** * @generated from message stickydisk.v1.GetStickyDiskRequest */ @@ -66085,6 +66265,8 @@ const GetStickyDiskResponse = /*@__PURE__*/ proto3.makeMessageType( { no: 5, name: "buildkitd_config", kind: "message", T: BuildkitdConfig }, { no: 6, name: "commit_early_deny", kind: "scalar", T: 8 /* ScalarType.BOOL */ }, { no: 7, name: "commit_early_deny_reason", kind: "scalar", T: 9 /* ScalarType.STRING */ }, + { no: 8, name: "fstrim_timeout_secs", kind: "scalar", T: 3 /* ScalarType.INT64 */ }, + { no: 9, name: "fstrim_min_extent_bytes", kind: "scalar", T: 3 /* ScalarType.INT64 */ }, ], ); @@ -66265,6 +66447,12 @@ const DockerJobLifecycle = /*@__PURE__*/ proto3.makeMessageType( { no: 19, name: "buildkitd_sigkill_used", kind: "scalar", T: 8 /* ScalarType.BOOL */ }, { no: 20, name: "history_export_timed_out", kind: "scalar", T: 8 /* ScalarType.BOOL */ }, { no: 21, name: "history_prune_failed", kind: "scalar", T: 8 /* ScalarType.BOOL */ }, + { no: 22, name: "timeline_bytes", kind: "scalar", T: 3 /* ScalarType.INT64 */ }, + { no: 23, name: "timeline_truncated", kind: "scalar", T: 8 /* ScalarType.BOOL */ }, + { no: 24, name: "history_export_bytes", kind: "scalar", T: 3 /* ScalarType.INT64 */ }, + { no: 25, name: "traces_dropped_oversize", kind: "scalar", T: 5 /* ScalarType.INT32 */ }, + { no: 26, name: "traces_dropped_payload_cap", kind: "scalar", T: 5 /* ScalarType.INT32 */ }, + { no: 27, name: "records_dropped_payload_cap", kind: "scalar", T: 5 /* ScalarType.INT32 */ }, ], ); @@ -66278,6 +66466,103 @@ const ReportDockerBuildResponse = /*@__PURE__*/ proto3.makeMessageType( ], ); +/** + * ContainerStartSpan is one container's create/start/die timing from the + * docker events stream, the read-side cost signal of the cache: docker + * pulls before create, so create-to-start never includes registry time for + * either class; it is the time to bring the container up from the layers + * on disk. Cached layers are cold reads from the Ceph-backed cache disk, + * pulled ones were just written and are page-cache warm, so the + * cached-vs-pulled split isolates the cache disk's read tax (the avoided + * pull is sized separately from image bytes). + * + * @generated from message stickydisk.v1.ContainerStartSpan + */ +const ContainerStartSpan = /*@__PURE__*/ proto3.makeMessageType( + "stickydisk.v1.ContainerStartSpan", + () => [ + { no: 1, name: "container_id", kind: "scalar", T: 9 /* ScalarType.STRING */ }, + { no: 2, name: "image_ref", kind: "scalar", T: 9 /* ScalarType.STRING */ }, + { no: 3, name: "image_id", kind: "scalar", T: 9 /* ScalarType.STRING */ }, + { no: 4, name: "serve_class", kind: "enum", T: proto3.getEnumType(ContainerServeClass) }, + { no: 5, name: "created_at_unix_ms", kind: "scalar", T: 3 /* ScalarType.INT64 */ }, + { no: 6, name: "started_at_unix_ms", kind: "scalar", T: 3 /* ScalarType.INT64 */ }, + { no: 7, name: "died_at_unix_ms", kind: "scalar", T: 3 /* ScalarType.INT64 */ }, + ], +); + +/** + * DockerContainerCacheLifecycle is the per-job container cache lifecycle + * summary: activation, boot-path costs, manifest maintenance results, + * working-set composition, shutdown, and the commit funnel. + * + * @generated from message stickydisk.v1.DockerContainerCacheLifecycle + */ +const DockerContainerCacheLifecycle = /*@__PURE__*/ proto3.makeMessageType( + "stickydisk.v1.DockerContainerCacheLifecycle", + () => [ + { no: 1, name: "setup_outcome", kind: "enum", T: proto3.getEnumType(ContainerCacheSetupOutcome) }, + { no: 2, name: "setup_duration_ms", kind: "scalar", T: 3 /* ScalarType.INT64 */ }, + { no: 3, name: "first_format", kind: "scalar", T: 8 /* ScalarType.BOOL */ }, + { no: 4, name: "daemon_start_ms", kind: "scalar", T: 3 /* ScalarType.INT64 */ }, + { no: 5, name: "cleanup_duration_ms", kind: "scalar", T: 3 /* ScalarType.INT64 */ }, + { no: 6, name: "manifest_update_ms", kind: "scalar", T: 3 /* ScalarType.INT64 */ }, + { no: 7, name: "manifest_update_failed", kind: "scalar", T: 8 /* ScalarType.BOOL */ }, + { no: 8, name: "manifest_rebuilt", kind: "scalar", T: 8 /* ScalarType.BOOL */ }, + { no: 9, name: "manifest_migrated", kind: "scalar", T: 8 /* ScalarType.BOOL */ }, + { no: 10, name: "overlay_gc_ms", kind: "scalar", T: 3 /* ScalarType.INT64 */ }, + { no: 11, name: "overlay_gc_bytes", kind: "scalar", T: 3 /* ScalarType.INT64 */ }, + { no: 12, name: "shutdown_ms", kind: "scalar", T: 3 /* ScalarType.INT64 */ }, + { no: 13, name: "working_set_images", kind: "scalar", T: 3 /* ScalarType.INT64 */ }, + { no: 14, name: "working_set_bytes", kind: "scalar", T: 3 /* ScalarType.INT64 */ }, + { no: 15, name: "probation_images", kind: "scalar", T: 3 /* ScalarType.INT64 */ }, + { no: 16, name: "probation_bytes", kind: "scalar", T: 3 /* ScalarType.INT64 */ }, + { no: 17, name: "promoted_images", kind: "scalar", T: 3 /* ScalarType.INT64 */ }, + { no: 18, name: "promoted_bytes", kind: "scalar", T: 3 /* ScalarType.INT64 */ }, + { no: 19, name: "evicted_images", kind: "scalar", T: 3 /* ScalarType.INT64 */ }, + { no: 20, name: "evicted_bytes", kind: "scalar", T: 3 /* ScalarType.INT64 */ }, + { no: 21, name: "evicted_probation_images", kind: "scalar", T: 3 /* ScalarType.INT64 */ }, + { no: 22, name: "evicted_probation_bytes", kind: "scalar", T: 3 /* ScalarType.INT64 */ }, + { no: 23, name: "reclaimable_bytes", kind: "scalar", T: 3 /* ScalarType.INT64 */ }, + { no: 24, name: "total_refs", kind: "scalar", T: 3 /* ScalarType.INT64 */ }, + { no: 25, name: "pruned_refs", kind: "scalar", T: 3 /* ScalarType.INT64 */ }, + { no: 26, name: "material_prune", kind: "scalar", T: 8 /* ScalarType.BOOL */ }, + { no: 27, name: "fs_used_bytes", kind: "scalar", T: 3 /* ScalarType.INT64 */ }, + { no: 28, name: "fs_size_bytes", kind: "scalar", T: 3 /* ScalarType.INT64 */ }, + { no: 29, name: "image_layer_bytes", kind: "scalar", T: 3 /* ScalarType.INT64 */ }, + { no: 31, name: "commit_reason", kind: "enum", T: proto3.getEnumType(ContainerCacheCommitReason) }, + { no: 32, name: "commit_post_outcome", kind: "enum", T: proto3.getEnumType(ContainerCacheCommitPostOutcome) }, + { no: 33, name: "current_container_hash", kind: "scalar", T: 9 /* ScalarType.STRING */ }, + { no: 34, name: "stored_container_hash", kind: "scalar", T: 9 /* ScalarType.STRING */ }, + { no: 35, name: "carryover_probation_images", kind: "scalar", T: 3 /* ScalarType.INT64 */ }, + { no: 36, name: "planned_prunes", kind: "scalar", T: 3 /* ScalarType.INT64 */ }, + { no: 37, name: "maintenance_budget_exhausted", kind: "scalar", T: 8 /* ScalarType.BOOL */ }, + { no: 38, name: "used_log_truncated", kind: "scalar", T: 8 /* ScalarType.BOOL */ }, + ], +); + +/** + * @generated from message stickydisk.v1.ReportDockerContainerCacheRequest + */ +const ReportDockerContainerCacheRequest = /*@__PURE__*/ proto3.makeMessageType( + "stickydisk.v1.ReportDockerContainerCacheRequest", + () => [ + { no: 1, name: "vm_id", kind: "scalar", T: 9 /* ScalarType.STRING */ }, + { no: 2, name: "expose_id", kind: "scalar", T: 9 /* ScalarType.STRING */ }, + { no: 3, name: "lifecycle", kind: "message", T: DockerContainerCacheLifecycle }, + { no: 4, name: "container_starts", kind: "message", T: ContainerStartSpan, repeated: true }, + { no: 5, name: "runner_step_timeline", kind: "scalar", T: 12 /* ScalarType.BYTES */ }, + ], +); + +/** + * @generated from message stickydisk.v1.ReportDockerContainerCacheResponse + */ +const ReportDockerContainerCacheResponse = /*@__PURE__*/ proto3.makeMessageType( + "stickydisk.v1.ReportDockerContainerCacheResponse", + [], +); + ;// CONCATENATED MODULE: ./node_modules/@bufbuild/protobuf/dist/esm/service-type.js // Copyright 2021-2024 Buf Technologies, Inc. @@ -66399,6 +66684,23 @@ const StickyDiskService = { O: ReportDockerBuildResponse, kind: MethodKind.Unary, }, + /** + * ReportDockerContainerCache is the structured docker container cache + * teardown report from the guest's job-completed hook: one lifecycle + * summary per job (setup/activation, manifest maintenance, eviction, + * shutdown, commit decision) plus per-container start spans captured from + * docker events. The guest ships only facts plus the expose_id from + * activation; all identity (sticky disk key, entity, installation, region, + * host, GitHub run/job IDs) is stamped host-side. + * + * @generated from rpc stickydisk.v1.StickyDiskService.ReportDockerContainerCache + */ + reportDockerContainerCache: { + name: "ReportDockerContainerCache", + I: ReportDockerContainerCacheRequest, + O: ReportDockerContainerCacheResponse, + kind: MethodKind.Unary, + }, } }; diff --git a/package-lock.json b/package-lock.json index 7129c5b..f90321e 100644 --- a/package-lock.json +++ b/package-lock.json @@ -14,8 +14,8 @@ "@actions/github": "^6.0.0", "@actions/io": "^1.1.3", "@actions/tool-cache": "^2.0.1", - "@buf/blacksmith_vm-agent.bufbuild_es": "1.9.0-20260904133914-c6beb298c70c.2", - "@buf/blacksmith_vm-agent.connectrpc_es": "1.6.1-20260904133914-c6beb298c70c.2", + "@buf/blacksmith_vm-agent.bufbuild_es": "1.9.0-20261023032817-36eedc7ae378.2", + "@buf/blacksmith_vm-agent.connectrpc_es": "1.6.1-20261023032817-36eedc7ae378.2", "@bufbuild/protobuf": "^1.9.0", "@connectrpc/connect": "^1.6.1", "@connectrpc/connect-node": "^1.6.1", @@ -628,25 +628,25 @@ "dev": true }, "node_modules/@buf/blacksmith_vm-agent.bufbuild_es": { - "version": "1.9.0-20260904133914-c6beb298c70c.2", - "resolved": "https://buf.build/gen/npm/v1/@buf/blacksmith_vm-agent.bufbuild_es/-/blacksmith_vm-agent.bufbuild_es-1.9.0-20260904133914-c6beb298c70c.2.tgz", + "version": "1.9.0-20261023032817-36eedc7ae378.2", + "resolved": "https://buf.build/gen/npm/v1/@buf/blacksmith_vm-agent.bufbuild_es/-/blacksmith_vm-agent.bufbuild_es-1.9.0-20261023032817-36eedc7ae378.2.tgz", "peerDependencies": { "@bufbuild/protobuf": "^1.9.0" } }, "node_modules/@buf/blacksmith_vm-agent.connectrpc_es": { - "version": "1.6.1-20260904133914-c6beb298c70c.2", - "resolved": "https://buf.build/gen/npm/v1/@buf/blacksmith_vm-agent.connectrpc_es/-/blacksmith_vm-agent.connectrpc_es-1.6.1-20260904133914-c6beb298c70c.2.tgz", + "version": "1.6.1-20261023032817-36eedc7ae378.2", + "resolved": "https://buf.build/gen/npm/v1/@buf/blacksmith_vm-agent.connectrpc_es/-/blacksmith_vm-agent.connectrpc_es-1.6.1-20261023032817-36eedc7ae378.2.tgz", "dependencies": { - "@buf/blacksmith_vm-agent.bufbuild_es": "1.10.0-20260904133914-c6beb298c70c.1" + "@buf/blacksmith_vm-agent.bufbuild_es": "1.10.0-20261023032817-36eedc7ae378.1" }, "peerDependencies": { "@connectrpc/connect": "^1.6.1" } }, "node_modules/@buf/blacksmith_vm-agent.connectrpc_es/node_modules/@buf/blacksmith_vm-agent.bufbuild_es": { - "version": "1.10.0-20260904133914-c6beb298c70c.1", - "resolved": "https://buf.build/gen/npm/v1/@buf/blacksmith_vm-agent.bufbuild_es/-/blacksmith_vm-agent.bufbuild_es-1.10.0-20260904133914-c6beb298c70c.1.tgz", + "version": "1.10.0-20261023032817-36eedc7ae378.1", + "resolved": "https://buf.build/gen/npm/v1/@buf/blacksmith_vm-agent.bufbuild_es/-/blacksmith_vm-agent.bufbuild_es-1.10.0-20261023032817-36eedc7ae378.1.tgz", "peerDependencies": { "@bufbuild/protobuf": "^1.10.0" } diff --git a/package.json b/package.json index be0a7a6..81951dd 100644 --- a/package.json +++ b/package.json @@ -33,8 +33,8 @@ "@actions/github": "^6.0.0", "@actions/io": "^1.1.3", "@actions/tool-cache": "^2.0.1", - "@buf/blacksmith_vm-agent.bufbuild_es": "1.9.0-20260904133914-c6beb298c70c.2", - "@buf/blacksmith_vm-agent.connectrpc_es": "1.6.1-20260904133914-c6beb298c70c.2", + "@buf/blacksmith_vm-agent.bufbuild_es": "1.9.0-20261023032817-36eedc7ae378.2", + "@buf/blacksmith_vm-agent.connectrpc_es": "1.6.1-20261023032817-36eedc7ae378.2", "@bufbuild/protobuf": "^1.9.0", "@connectrpc/connect": "^1.6.1", "@connectrpc/connect-node": "^1.6.1", diff --git a/src/blacksmith-cache.ts b/src/blacksmith-cache.ts index f073efc..2664275 100644 --- a/src/blacksmith-cache.ts +++ b/src/blacksmith-cache.ts @@ -116,6 +116,13 @@ export interface CacheInfo { // sticky disk commit will be denied (e.g. branch protection): the mirror can // be used but changes to it are discarded at VM teardown. commitEarlyDenyReason: string + // fstrimTimeoutSecs is the host's budget, in whole seconds, for trimming the + // mirror filesystem before unmount at commit; 0 when the host has not + // enabled the trim. + fstrimTimeoutSecs: number + // fstrimMinExtentBytes is the host's `fstrim -m` value in bytes; 0 means + // the fstrim default (every free extent). + fstrimMinExtentBytes: number } /** @@ -364,7 +371,9 @@ export async function setupCache( hydrationInProgress: true, hydrationMessage, performedHydration: false, - commitEarlyDenyReason: '' + commitEarlyDenyReason: '', + fstrimTimeoutSecs: 0, + fstrimMinExtentBytes: 0 } } // Re-throw other errors @@ -391,6 +400,10 @@ export async function setupCache( const commitEarlyDenyReason = response.commitEarlyDeny ? response.commitEarlyDenyReason || 'denied by host policy' : '' + const fstrimTimeoutSecs = parseFstrimTimeoutSecs(response.fstrimTimeoutSecs) + const fstrimMinExtentBytes = parseFstrimMinExtentBytes( + response.fstrimMinExtentBytes + ) if (commitEarlyDenyReason) { core.notice( `[git-mirror] Sticky disk changes will not be committed for this job (${commitEarlyDenyReason}). The git mirror cache is used as-is and any changes to it are discarded.` @@ -418,7 +431,9 @@ export async function setupCache( mirrorPath: getMirrorPath(owner, repo), hydrationInProgress: false, performedHydration: false, // Will be set by ensureMirror if we do initial clone - commitEarlyDenyReason + commitEarlyDenyReason, + fstrimTimeoutSecs, + fstrimMinExtentBytes } } @@ -2315,6 +2330,74 @@ async function flushBlockDevice(devicePath: string): Promise { } } +/** + * Parse the host's fstrim budget (GetStickyDiskResponse.fstrim_timeout_secs, + * a protobuf int64 that arrives as a bigint, or the string read back from + * action state) into whole seconds. Anything unusable is 0, which disables + * the trim. + */ +export function parseFstrimTimeoutSecs(value: unknown): number { + const n = typeof value === 'bigint' ? Number(value) : Number(value ?? 0) + if (!Number.isFinite(n) || n <= 0) return 0 + return Math.floor(n) +} + +/** + * Parse the host's `fstrim --minimum` value + * (GetStickyDiskResponse.fstrim_min_extent_bytes, a bigint or saved-state + * string) into whole bytes. Anything unusable is 0: the fstrim default. + */ +export function parseFstrimMinExtentBytes(value: unknown): number { + return parseFstrimTimeoutSecs(value) +} + +/** + * Best-effort `fstrim` of the mirror filesystem before it is unmounted and + * committed: deleted packs keep their RBD extents allocated on the backing + * store until the guest discards them. Bounded by the host-supplied budget; + * any failure (discard not negotiated, missing binary, timeout) only logs. + */ +async function trimFilesystem( + mountPoint: string, + timeoutSecs: number, + minExtentBytes: number +): Promise { + const start = Date.now() + try { + const result = await exec.getExecOutput( + 'timeout', + [ + String(timeoutSecs), + 'sudo', + 'fstrim', + '-v', + ...(minExtentBytes > 0 ? ['-m', String(minExtentBytes)] : []), + '--', + mountPoint + ], + {ignoreReturnCode: true, silent: true} + ) + const durationMs = Date.now() - start + if (result.exitCode === 0) { + core.info( + `[git-mirror] fstrim ${mountPoint}: ${result.stdout.trim() || 'done'} in ${durationMs}ms` + ) + } else if (result.exitCode === TIMEOUT_EXIT_CODE) { + core.info( + `[git-mirror] fstrim ${mountPoint}: timed out after ${timeoutSecs}s` + ) + } else { + core.info( + `[git-mirror] fstrim ${mountPoint}: failed in ${durationMs}ms: ${result.stderr.trim()}` + ) + } + } catch (error) { + core.info( + `[git-mirror] fstrim ${mountPoint}: failed: ${(error as Error).message}` + ) + } +} + export interface CleanupOptions { exposeId: string stickyDiskKey: string @@ -2332,12 +2415,16 @@ export interface CleanupOptions { // Mirror sync outcome from the main step. mirrorSyncFailed?: boolean mirrorSyncTimedOut?: boolean + // Host-supplied fstrim budget in whole seconds; 0 or absent skips the trim. + fstrimTimeoutSecs?: number + // Host-supplied `fstrim -m` value in bytes; 0 or absent uses the default. + fstrimMinExtentBytes?: number } /** * Cleanup: run mirror maintenance, sync, unmount, and commit the sticky disk. * - * Execution order: maintenance → sync → unmount (with retry) → flush → commit + * Execution order: maintenance → sync → fstrim → unmount (with retry) → flush → commit * If the mirror sync failed or timed out, shouldCommit is set to false. * Maintenance failure or timeout does not affect the commit. */ @@ -2350,7 +2437,9 @@ export async function cleanup(options: CleanupOptions): Promise { mirrorPath, mirrorSyncFailed, mirrorSyncTimedOut, - vmHydratedGitMirror + vmHydratedGitMirror, + fstrimTimeoutSecs, + fstrimMinExtentBytes } = options let {shouldCommit} = options @@ -2394,6 +2483,22 @@ export async function cleanup(options: CleanupOptions): Promise { core.warning('[git-mirror] Failed to sync filesystem') } + // Discard freed blocks before unmount so the committed snapshot does not + // keep them allocated. Only worth it when the result is persisted, and only + // when the host has enabled it. + if ( + mountPoint && + shouldCommit && + fstrimTimeoutSecs && + fstrimTimeoutSecs > 0 + ) { + await trimFilesystem( + mountPoint, + fstrimTimeoutSecs, + fstrimMinExtentBytes ?? 0 + ) + } + // Get device path before unmount for durability flush let devicePath: string | null = null if (mountPoint) { diff --git a/src/git-source-provider.ts b/src/git-source-provider.ts index f4c9d45..b14dfa3 100644 --- a/src/git-source-provider.ts +++ b/src/git-source-provider.ts @@ -186,6 +186,12 @@ async function getSourceInner( stateHelper.setBlacksmithCacheCommitEarlyDenyReason( cacheInfo.commitEarlyDenyReason ) + stateHelper.setBlacksmithCacheFstrimTimeoutSecs( + cacheInfo.fstrimTimeoutSecs + ) + stateHelper.setBlacksmithCacheFstrimMinExtentBytes( + cacheInfo.fstrimMinExtentBytes + ) const mirrorExisted = fsHelper.directoryExistsSync( cacheInfo.mirrorPath diff --git a/src/main.ts b/src/main.ts index f875dc9..ebb99fa 100644 --- a/src/main.ts +++ b/src/main.ts @@ -154,7 +154,9 @@ async function cleanup(): Promise { shouldCommit, vmHydratedGitMirror: performedHydration, mirrorSyncFailed, - mirrorSyncTimedOut + mirrorSyncTimedOut, + fstrimTimeoutSecs: stateHelper.BlacksmithCacheFstrimTimeoutSecs, + fstrimMinExtentBytes: stateHelper.BlacksmithCacheFstrimMinExtentBytes }) } catch (error) { core.warning( diff --git a/src/state-helper.ts b/src/state-helper.ts index e420115..3f962ea 100644 --- a/src/state-helper.ts +++ b/src/state-helper.ts @@ -106,6 +106,24 @@ export const BlacksmithCacheCommitEarlyDenyReason = core.getState( 'blacksmithCacheCommitEarlyDenyReason' ) +/** + * Host-supplied budget, in whole seconds, for trimming the mirror filesystem + * before unmount at commit. 0 when the host has not enabled the trim. + */ +export const BlacksmithCacheFstrimTimeoutSecs = (() => { + const secs = Number(core.getState('blacksmithCacheFstrimTimeoutSecs') || '0') + return Number.isFinite(secs) && secs > 0 ? Math.floor(secs) : 0 +})() + +/** + * The host's `fstrim -m` value in bytes, saved by the main action. 0 when + * the host left fstrim at its default minimum extent. + */ +export const BlacksmithCacheFstrimMinExtentBytes = (() => { + const n = Number(core.getState('blacksmithCacheFstrimMinExtentBytes') || '0') + return Number.isFinite(n) && n > 0 ? Math.floor(n) : 0 +})() + /** * Save the repository path so the POST action can retrieve the value. */ @@ -233,6 +251,20 @@ export function setBlacksmithCacheCommitEarlyDenyReason(reason: string) { core.saveState('blacksmithCacheCommitEarlyDenyReason', reason) } +/** + * Save the host's fstrim budget so the POST action can trim before unmount. + */ +export function setBlacksmithCacheFstrimTimeoutSecs(secs: number) { + core.saveState('blacksmithCacheFstrimTimeoutSecs', String(secs)) +} + +/** + * Save the host's `fstrim -m` value so the POST action can pass it along. + */ +export function setBlacksmithCacheFstrimMinExtentBytes(bytes: number) { + core.saveState('blacksmithCacheFstrimMinExtentBytes', String(bytes)) +} + // Publish a variable so that when the POST action runs, it can determine it should run the cleanup logic. // This is necessary since we don't have a separate entry point. if (!IsPost) {