diff --git a/docs/guides/packaging.md b/docs/guides/packaging.md index 7644e597a7..87969a31a0 100644 --- a/docs/guides/packaging.md +++ b/docs/guides/packaging.md @@ -68,6 +68,10 @@ package: - '!.git/**' ``` +### Size limit + +AWS Lambda limits a function's code and layers to 250 MB once unzipped. When a package built by osls is larger than that, `serverless package` and `serverless deploy` fail before anything is uploaded, and list the largest files and directories of the package. Use `patterns` to exclude files that are not needed at runtime. + ### Artifact For complete control over the packaging process you can specify your own artifact zip file. diff --git a/lib/plugins/package/lib/zip-service.js b/lib/plugins/package/lib/zip-service.js index 711ef3a817..0c544c8c46 100644 --- a/lib/plugins/package/lib/zip-service.js +++ b/lib/plugins/package/lib/zip-service.js @@ -55,6 +55,10 @@ const getArchiveEntryName = (filePath, prefix) => { const zipMtime = new Date(0); const maxConcurrentFileStats = 64; +// AWS Lambda rejects packages larger than 250 MB (262144000 bytes) once unzipped +const maxUnzippedSize = 250 * 1024 * 1024; +const packagingDocsUrl = + 'https://github.com/oss-serverless/osls/blob/3.x/docs/guides/packaging.md#patterns'; const getFileMode = (stat) => Number(stat.mode) & 0o100 || process.platform === 'win32' ? 0o100755 : 0o100644; @@ -100,6 +104,43 @@ const assertSameFile = (entry, currentStat) => { } }; +// Lambda counts MB as 1,024 KB +const formatMegabytes = (bytes) => `${(bytes / 1024 / 1024).toFixed(1)} MB`; + +// Fail before zipping instead of letting the deployment fail minutes later in CloudFormation +const assertUnzippedSizeWithinLimit = (entries, zipFileName) => { + const totalSize = entries.reduce((total, entry) => total + Number(entry.stat.size), 0); + if (totalSize < maxUnzippedSize) return; + + const sizeByTopLevelPath = new Map(); + for (const entry of entries) { + const separatorIndex = entry.name.indexOf('/'); + const topLevelPath = + separatorIndex === -1 ? entry.name : entry.name.slice(0, separatorIndex + 1); + sizeByTopLevelPath.set( + topLevelPath, + (sizeByTopLevelPath.get(topLevelPath) || 0) + Number(entry.stat.size) + ); + } + const largestPaths = Array.from(sizeByTopLevelPath) + .sort((a, b) => b[1] - a[1]) + .slice(0, 5); + const pathColumnWidth = Math.max(...largestPaths.map(([topLevelPath]) => topLevelPath.length)); + + throw new ServerlessError( + [ + `The "${zipFileName}" package is ${formatMegabytes(totalSize)} unzipped, above the 250 MB AWS Lambda limit (function code and layers combined).`, + 'Largest files and directories in the package:', + ...largestPaths.map( + ([topLevelPath, size]) => + ` ${topLevelPath.padEnd(pathColumnWidth)} ${formatMegabytes(size).padStart(9)}` + ), + `Exclude files that are not needed at runtime with "package.patterns": ${packagingDocsUrl}`, + ].join('\n'), + 'PACKAGE_TOO_LARGE' + ); +}; + const compareUtf8 = (left, right) => Buffer.compare(Buffer.from(left, 'utf8'), Buffer.from(right, 'utf8')); @@ -319,6 +360,12 @@ const zipService = { entries.sort(compareZipEntries); + // A getFileContent() override can change file sizes, the size on disk is only + // known to be the unzipped size when files are zipped as they are + if (usesDefaultGetFileContent) { + assertUnzippedSizeWithinLimit(entries, zipFileName); + } + for (const entry of entries) { const options = { mode: entry.mode, diff --git a/test/unit/lib/plugins/package/lib/zip-service.test.js b/test/unit/lib/plugins/package/lib/zip-service.test.js index 6a8ded9338..4c1ce56956 100644 --- a/test/unit/lib/plugins/package/lib/zip-service.test.js +++ b/test/unit/lib/plugins/package/lib/zip-service.test.js @@ -1397,7 +1397,8 @@ describe('zipService', () => { serverless.utils.writeFileDir(largeFilePath); await fs.promises.writeFile(largeFilePath, ''); - await fs.promises.truncate(largeFilePath, 512 * 1024 * 1024); + // Just below the 250 MB AWS Lambda limit, larger packages are rejected + await fs.promises.truncate(largeFilePath, 240 * 1024 * 1024); await packagePlugin.zipFiles(['event.json'], getTestArtifactFileName('warmup')); forceGc(); @@ -1416,8 +1417,8 @@ describe('zipService', () => { } // The sampled peak includes dead stream chunks that V8 has not collected yet, - // and that GC lag alone can exceed 64 MiB. The cap only needs to stay far - // below the 512 MiB file size to prove the file was streamed, not materialized. + // and that GC lag alone can exceed 64 MiB. The cap only needs to stay well + // below the 240 MiB file size to prove the file was streamed, not materialized. expect(peakArrayBuffers - before.arrayBuffers).to.be.lessThan(128 * 1024 * 1024); forceGc(); @@ -1435,6 +1436,40 @@ describe('zipService', () => { 'No files to package' )); + it('rejects packages above the AWS Lambda unzipped size limit before zipping them', async () => { + serverless.utils.writeFileSync(path.join(tmpDirPath, 'handler.js'), 'handler'); + // Sparse files: only their size is read + const largeFiles = { + 'vendor/large.bin': 200 * 1024 * 1024, + 'node_modules/medium.bin': 60 * 1024 * 1024, + }; + for (const [filePath, size] of Object.entries(largeFiles)) { + serverless.utils.writeFileSync(path.join(tmpDirPath, filePath), ''); + await fs.promises.truncate(path.join(tmpDirPath, filePath), size); + } + const artifactFilePath = path.join(serverless.serviceDir, '.serverless', 'too-large.zip'); + + let error; + try { + await packagePlugin.zipFiles(['handler.js', ...Object.keys(largeFiles)], 'too-large.zip'); + } catch (caughtError) { + error = caughtError; + } + + expect(error).to.have.property('code', 'PACKAGE_TOO_LARGE'); + expect(error.message).to.equal( + [ + 'The "too-large.zip" package is 260.0 MB unzipped, above the 250 MB AWS Lambda limit (function code and layers combined).', + 'Largest files and directories in the package:', + ' vendor/ 200.0 MB', + ' node_modules/ 60.0 MB', + ' handler.js 0.0 MB', + 'Exclude files that are not needed at runtime with "package.patterns": https://github.com/oss-serverless/osls/blob/3.x/docs/guides/packaging.md#patterns', + ].join('\n') + ); + expect(fs.existsSync(artifactFilePath)).to.equal(false); + }); + it('configures metadata stat concurrency with ext/promise/limit', () => { let configuredLimit; const fakeLimit = function (limitValue, callback) {