From cceb7826fc31848a4675533db497583d0ca4a825 Mon Sep 17 00:00:00 2001 From: Maik Schulze Date: Fri, 6 Nov 2026 12:05:01 +0100 Subject: [PATCH] gh-161478: Collect large compression outputs in blocks again, except with glibc Since gh-139877, _BlocksOutputBuffer grows one PyBytesWriter with realloc(). On macOS and Windows, a large block grown with realloc() lands in memory the process has never touched (Windows also copies it), so every call faults in the output's pages again, and zlib, gzip, lzma and zstd decompression got up to 3x slower than in 3.14 from a few MiB of output, and on macOS the process kept the memory of every call. The first blocks (up to 1.34 MiB) still grow the writer. Later blocks are separate bytes objects, joined once at the end, as in 3.14, except on Linux with glibc in the default build, where realloc() grows a large block in place (mremap()) and the writer is faster. Platforms that were not measured, Android, and the free-threaded build (mimalloc copies on realloc()) get 3.14's behaviour. Tests: outputs at and around the block boundaries, for every module. --- .../internal/pycore_blocks_output_buffer.h | 148 +++++++++++++++--- Lib/test/test_bz2.py | 44 ++++++ Lib/test/test_gzip.py | 23 +++ Lib/test/test_lzma.py | 44 ++++++ Lib/test/test_zlib.py | 52 ++++++ Lib/test/test_zstd.py | 44 ++++++ ...-11-05-21-00-00.gh-issue-161478.OutBuf.rst | 4 + 7 files changed, 340 insertions(+), 19 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2026-11-05-21-00-00.gh-issue-161478.OutBuf.rst diff --git a/Include/internal/pycore_blocks_output_buffer.h b/Include/internal/pycore_blocks_output_buffer.h index 322c1e93344ba30..94ab0cbe2d60d9f 100644 --- a/Include/internal/pycore_blocks_output_buffer.h +++ b/Include/internal/pycore_blocks_output_buffer.h @@ -6,14 +6,17 @@ stream->next_out: point to the next output position. stream->avail_out: the number of available bytes left in the buffer. - It maintains a list of bytes object, so there is no overhead of resizing - the buffer. + The first blocks are written into a PyBytesWriter, which is returned as is + when the output fits in them. Further blocks are separate bytes objects, + joined once by _BlocksOutputBuffer_Finish(), except where the writer is + known to grow in place (glibc). See OUTPUT_BUFFER_HEAD_SIZE. Usage: 1, Initialize the struct instance like this: - _BlocksOutputBuffer buffer = {.list = NULL}; - Set .list to NULL for _BlocksOutputBuffer_OnError() + _BlocksOutputBuffer buffer = {.writer = NULL}; + This zeroes the other members too. _BlocksOutputBuffer_OnError() + relies on .writer and .list being NULL. 2, Initialize the buffer use one of these functions: _BlocksOutputBuffer_InitAndGrow() @@ -28,7 +31,8 @@ 5, Finish the buffer, and return a bytes object: _BlocksOutputBuffer_Finish() - 6, Clean up the buffer when an error occurred: + 6, Clean up the buffer when an error occurred, including when + _BlocksOutputBuffer_Finish() returned NULL: _BlocksOutputBuffer_OnError() */ @@ -53,6 +57,11 @@ typedef struct { Py_ssize_t max_length; // Number of blocks of bytes. Used to calculate next allocation size size_t num_blocks; + // Blocks after the first OUTPUT_BUFFER_HEAD_SIZE bytes, as a list of + // bytes objects. NULL while all the output is in the writer. + PyObject *list; + // Size of the writer's data once blocks are added to the list + Py_ssize_t head_size; } _BlocksOutputBuffer; static const char unable_allocate_msg[] = "Unable to allocate output buffer."; @@ -60,16 +69,46 @@ static const char unable_allocate_msg[] = "Unable to allocate output buffer."; /* In 32-bit build, the max block size should <= INT32_MAX. */ #define OUTPUT_BUFFER_MAX_BLOCK_SIZE (256*1024*1024) -/* Block size sequence */ -#define KB (1024) +/* Block size sequence. The first four blocks are the writer's head, see + OUTPUT_BUFFER_HEAD_SIZE; the tests in test_zlib, test_gzip, test_bz2, + test_lzma and test_zstd probe the boundaries of these sizes. */ +#define OUTPUT_BUFFER_BLOCK_1 (32*1024) +#define OUTPUT_BUFFER_BLOCK_2 (64*1024) +#define OUTPUT_BUFFER_BLOCK_3 (256*1024) +#define OUTPUT_BUFFER_BLOCK_4 (1024*1024) #define MB (1024*1024) static const Py_ssize_t BUFFER_BLOCK_SIZE[] = - { 32*KB, 64*KB, 256*KB, 1*MB, 4*MB, 8*MB, 16*MB, 16*MB, + { OUTPUT_BUFFER_BLOCK_1, OUTPUT_BUFFER_BLOCK_2, OUTPUT_BUFFER_BLOCK_3, + OUTPUT_BUFFER_BLOCK_4, 4*MB, 8*MB, 16*MB, 16*MB, 32*MB, 32*MB, 32*MB, 32*MB, 64*MB, 64*MB, 128*MB, 128*MB, OUTPUT_BUFFER_MAX_BLOCK_SIZE }; -#undef KB #undef MB +/* The writer grows to at most this many bytes: the first four blocks of the + sequence, 1.34 MiB. Up to there, returning the writer without a join + beats separate blocks; from the 4 MiB block on, the blocks win (measured + on macOS and Windows, see gh-161478). Later blocks go to the list, and so + does every block after a larger initial size. + + On macOS and Windows, a large block grown with realloc() lands in memory + the process has never touched (on Windows, realloc() also copies it), so + a writer that keeps growing faults in new pages on every call, which + costs more than writing blocks once and joining them at the end, as the + code before gh-139877 did. With glibc, realloc() grows a large block in + place (mremap()), and there growing the writer is faster than joining + blocks. Everything else gets the blocks: platforms that were not + measured, and allocators that copy on realloc(): Android's, musl's, and + mimalloc, the free-threaded build's object allocator (with + PYTHONMALLOC=mimalloc on macOS, the growing writer regressed just like + with the system allocator). */ +#if defined(__GLIBC__) && !defined(Py_GIL_DISABLED) +# define OUTPUT_BUFFER_HEAD_SIZE PY_SSIZE_T_MAX +#else +# define OUTPUT_BUFFER_HEAD_SIZE \ + (OUTPUT_BUFFER_BLOCK_1 + OUTPUT_BUFFER_BLOCK_2 \ + + OUTPUT_BUFFER_BLOCK_3 + OUTPUT_BUFFER_BLOCK_4) +#endif + /* According to the block sizes defined by BUFFER_BLOCK_SIZE, the whole allocated size growth step is: 1 32 KB +32 KB @@ -130,6 +169,8 @@ _BlocksOutputBuffer_InitAndGrow(_BlocksOutputBuffer *buffer, buffer->allocated = block_size; buffer->max_length = max_length; buffer->num_blocks = 1; + buffer->list = NULL; + buffer->head_size = 0; *next_out = PyBytesWriter_GetData(buffer->writer); return block_size; @@ -161,6 +202,8 @@ _BlocksOutputBuffer_InitWithSize(_BlocksOutputBuffer *buffer, buffer->allocated = init_size; buffer->max_length = -1; buffer->num_blocks = 1; + buffer->list = NULL; + buffer->head_size = 0; *next_out = PyBytesWriter_GetData(buffer->writer); return init_size; @@ -209,19 +252,44 @@ _BlocksOutputBuffer_Grow(_BlocksOutputBuffer *buffer, return -1; } - if (PyBytesWriter_Grow(buffer->writer, block_size)) { - PyErr_SetString(PyExc_MemoryError, unable_allocate_msg); - return -1; + if (buffer->list == NULL + && buffer->allocated <= OUTPUT_BUFFER_HEAD_SIZE - block_size) + { + // grow the writer + if (PyBytesWriter_Grow(buffer->writer, block_size)) { + PyErr_SetString(PyExc_MemoryError, unable_allocate_msg); + return -1; + } + char *data = PyBytesWriter_GetData(buffer->writer); + *next_out = data + buffer->allocated; + } + else { + // add a block to the list + if (buffer->list == NULL) { + buffer->list = PyList_New(0); + if (buffer->list == NULL) { + PyErr_SetString(PyExc_MemoryError, unable_allocate_msg); + return -1; + } + buffer->head_size = buffer->allocated; + } + PyObject *block = PyBytes_FromStringAndSize(NULL, block_size); + if (block == NULL) { + PyErr_SetString(PyExc_MemoryError, unable_allocate_msg); + return -1; + } + if (PyList_Append(buffer->list, block) < 0) { + Py_DECREF(block); + PyErr_SetString(PyExc_MemoryError, unable_allocate_msg); + return -1; + } + Py_DECREF(block); // the list holds it + *next_out = PyBytes_AS_STRING(block); } - - Py_ssize_t current_size = buffer->allocated; // set variables buffer->allocated += block_size; buffer->num_blocks += 1; - - char *data = PyBytesWriter_GetData(buffer->writer); - *next_out = data + current_size; return block_size; } @@ -244,9 +312,50 @@ _BlocksOutputBuffer_Finish(_BlocksOutputBuffer *buffer, { PyObject *obj; assert(buffer->writer != NULL); - obj = PyBytesWriter_FinishWithSize(buffer->writer, - buffer->allocated - avail_out); + if (buffer->list == NULL) { + obj = PyBytesWriter_FinishWithSize(buffer->writer, + buffer->allocated - avail_out); + buffer->writer = NULL; + return obj; + } + + Py_ssize_t num = PyList_GET_SIZE(buffer->list); + if (num == 1 + && avail_out == PyBytes_GET_SIZE(PyList_GET_ITEM(buffer->list, 0))) + { + // the one block was never written to: the writer holds all the data + obj = PyBytesWriter_FinishWithSize(buffer->writer, buffer->head_size); + buffer->writer = NULL; + Py_CLEAR(buffer->list); + return obj; + } + + // join the writer's data and the blocks into one bytes object + Py_ssize_t size = buffer->allocated - avail_out; + obj = PyBytes_FromStringAndSize(NULL, size); + if (obj == NULL) { + PyErr_SetString(PyExc_MemoryError, unable_allocate_msg); + return NULL; // the caller calls _BlocksOutputBuffer_OnError() + } + char *dst = PyBytes_AS_STRING(obj); + memcpy(dst, PyBytesWriter_GetData(buffer->writer), buffer->head_size); + dst += buffer->head_size; + + for (Py_ssize_t i = 0; i < num; i++) { + PyObject *block = PyList_GET_ITEM(buffer->list, i); + Py_ssize_t block_size = PyBytes_GET_SIZE(block); + if (i == num - 1) { + // the last block holds avail_out unused bytes + block_size -= avail_out; + } + memcpy(dst, PyBytes_AS_STRING(block), block_size); + dst += block_size; + } + assert(dst == PyBytes_AS_STRING(obj) + size); + + PyBytesWriter_Discard(buffer->writer); buffer->writer = NULL; + Py_CLEAR(buffer->list); return obj; } @@ -256,6 +365,7 @@ _BlocksOutputBuffer_OnError(_BlocksOutputBuffer *buffer) { PyBytesWriter_Discard(buffer->writer); buffer->writer = NULL; + Py_CLEAR(buffer->list); } #ifdef __cplusplus diff --git a/Lib/test/test_bz2.py b/Lib/test/test_bz2.py index 6e0a1207ddfc186..10c12c21d3ee673 100644 --- a/Lib/test/test_bz2.py +++ b/Lib/test/test_bz2.py @@ -1324,6 +1324,50 @@ def test_bzlib_version(self): self.assertStartsWith(bz2.bzlib_version, '%d.%d.%d' % v) +class OutputBufferTest(unittest.TestCase): + # Outputs at and around the boundaries of the internal output buffer's + # blocks (Include/internal/pycore_blocks_output_buffer.h): the first four + # (32 KiB, 64 KiB, 256 KiB and 1 MiB) are one buffer grown by realloc(); + # the next one (4 MiB) is a separate block on most platforms. + HEAD = (32 + 64 + 256 + 1024) * 1024 + SIZES = [HEAD - 1, HEAD, HEAD + 1, + HEAD + 4 * 1024 * 1024 - 1, HEAD + 4 * 1024 * 1024, + HEAD + 4 * 1024 * 1024 + 1] + # a pattern every codec compresses quickly: 256 varied bytes per 4 KiB + PATTERN = bytes(range(256)) + bytes(4096 - 256) + + @classmethod + def data(cls, size): + return (cls.PATTERN * (size // len(cls.PATTERN) + 1))[:size] + + @classmethod + def setUpClass(cls): + cls.compressed = {size: bz2.compress(cls.data(size), 1) + for size in cls.SIZES} + + def test_decompress(self): + for size in self.SIZES: + with self.subTest(size=size): + data = self.data(size) + self.assertEqual(bz2.decompress(self.compressed[size]), data) + + def test_decompressor(self): + for size in self.SIZES: + with self.subTest(size=size): + data = self.data(size) + compressed = self.compressed[size] + self.assertEqual( + BZ2Decompressor().decompress(compressed), data) + # max_length past the first blocks and short of the next one + max_length = self.HEAD + 100 * 1024 + d = BZ2Decompressor() + first = d.decompress(compressed, max_length) + self.assertEqual(len(first), min(size, max_length)) + rest = b'' if d.eof else d.decompress(b'') + self.assertEqual(first + rest, data) + self.assertTrue(d.eof) + + class TestModule(unittest.TestCase): def test_deprecated_module_attributes(self): support.check_deprecated_module_attributes( diff --git a/Lib/test/test_gzip.py b/Lib/test/test_gzip.py index b0706fdcabfdc8c..3cdbe3852c28715 100644 --- a/Lib/test/test_gzip.py +++ b/Lib/test/test_gzip.py @@ -1337,5 +1337,28 @@ def test_decompress_cannot_have_flags_compression(self): self.assertEqual(out, b'') +class TestOutputBuffer(unittest.TestCase): + # Outputs at and around the boundaries of the internal output buffer's + # blocks (Include/internal/pycore_blocks_output_buffer.h): the first four + # (32 KiB, 64 KiB, 256 KiB and 1 MiB) are one buffer grown by realloc(); + # the next one (4 MiB) is a separate block on most platforms. + HEAD = (32 + 64 + 256 + 1024) * 1024 + SIZES = [HEAD - 1, HEAD, HEAD + 1, + HEAD + 4 * 1024 * 1024 - 1, HEAD + 4 * 1024 * 1024, + HEAD + 4 * 1024 * 1024 + 1] + # a pattern every codec compresses quickly: 256 varied bytes per 4 KiB + PATTERN = bytes(range(256)) + bytes(4096 - 256) + + @classmethod + def data(cls, size): + return (cls.PATTERN * (size // len(cls.PATTERN) + 1))[:size] + + def test_decompress(self): + for size in self.SIZES: + with self.subTest(size=size): + data = self.data(size) + self.assertEqual(gzip.decompress(gzip.compress(data, 1)), data) + + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_lzma.py b/Lib/test/test_lzma.py index c14b3ebf586c96f..e6f2d313ebe84fe 100644 --- a/Lib/test/test_lzma.py +++ b/Lib/test/test_lzma.py @@ -2214,5 +2214,49 @@ def test_filter_properties_roundtrip(self): ) +class OutputBufferTestCase(unittest.TestCase): + # Outputs at and around the boundaries of the internal output buffer's + # blocks (Include/internal/pycore_blocks_output_buffer.h): the first four + # (32 KiB, 64 KiB, 256 KiB and 1 MiB) are one buffer grown by realloc(); + # the next one (4 MiB) is a separate block on most platforms. + HEAD = (32 + 64 + 256 + 1024) * 1024 + SIZES = [HEAD - 1, HEAD, HEAD + 1, + HEAD + 4 * 1024 * 1024 - 1, HEAD + 4 * 1024 * 1024, + HEAD + 4 * 1024 * 1024 + 1] + # a pattern every codec compresses quickly: 256 varied bytes per 4 KiB + PATTERN = bytes(range(256)) + bytes(4096 - 256) + + @classmethod + def data(cls, size): + return (cls.PATTERN * (size // len(cls.PATTERN) + 1))[:size] + + @classmethod + def setUpClass(cls): + cls.compressed = {size: lzma.compress(cls.data(size), preset=0) + for size in cls.SIZES} + + def test_decompress(self): + for size in self.SIZES: + with self.subTest(size=size): + data = self.data(size) + self.assertEqual(lzma.decompress(self.compressed[size]), data) + + def test_decompressor(self): + for size in self.SIZES: + with self.subTest(size=size): + data = self.data(size) + compressed = self.compressed[size] + self.assertEqual( + LZMADecompressor().decompress(compressed), data) + # max_length past the first blocks and short of the next one + max_length = self.HEAD + 100 * 1024 + d = LZMADecompressor() + first = d.decompress(compressed, max_length) + self.assertEqual(len(first), min(size, max_length)) + rest = b'' if d.eof else d.decompress(b'') + self.assertEqual(first + rest, data) + self.assertTrue(d.eof) + + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_zlib.py b/Lib/test/test_zlib.py index 3243448043b5c21..17ede69214585f9 100644 --- a/Lib/test/test_zlib.py +++ b/Lib/test/test_zlib.py @@ -1311,6 +1311,58 @@ def __index__(self): return 100 +class OutputBufferTestCase(unittest.TestCase): + # Outputs at and around the boundaries of the internal output buffer's + # blocks (Include/internal/pycore_blocks_output_buffer.h): the first four + # (32 KiB, 64 KiB, 256 KiB and 1 MiB) are one buffer grown by realloc(); + # the next one (4 MiB) is a separate block on most platforms. + HEAD = (32 + 64 + 256 + 1024) * 1024 + SIZES = [HEAD - 1, HEAD, HEAD + 1, + HEAD + 4 * 1024 * 1024 - 1, HEAD + 4 * 1024 * 1024, + HEAD + 4 * 1024 * 1024 + 1] + # a pattern every codec compresses quickly: 256 varied bytes per 4 KiB + PATTERN = bytes(range(256)) + bytes(4096 - 256) + + @classmethod + def data(cls, size): + return (cls.PATTERN * (size // len(cls.PATTERN) + 1))[:size] + + @classmethod + def setUpClass(cls): + cls.compressed = {size: zlib.compress(cls.data(size), 1) + for size in cls.SIZES} + + def test_decompress(self): + for size in self.SIZES: + with self.subTest(size=size): + data = self.data(size) + compressed = self.compressed[size] + # decompress() starts its buffer at bufsize; 32 KiB makes + # the blocks the ones SIZES is derived from + self.assertEqual( + zlib.decompress(compressed, bufsize=32 * 1024), data) + self.assertEqual(zlib.decompress(compressed), data) + # an initial buffer larger than the first blocks, which + # switches to separate blocks at its first growth + self.assertEqual( + zlib.decompress(compressed, bufsize=2 * 1024 * 1024), data) + + def test_decompressobj(self): + for size in self.SIZES: + with self.subTest(size=size): + data = self.data(size) + compressed = self.compressed[size] + d = zlib.decompressobj() + self.assertEqual(d.decompress(compressed) + d.flush(), data) + # max_length past the first blocks and short of the next one + max_length = self.HEAD + 100 * 1024 + d = zlib.decompressobj() + first = d.decompress(compressed, max_length) + self.assertEqual(len(first), min(size, max_length)) + rest = d.decompress(d.unconsumed_tail) + d.flush() + self.assertEqual(first + rest, data) + + class TestModule(unittest.TestCase): def test_deprecated_module_attributes(self): support.check_deprecated_module_attributes( diff --git a/Lib/test/test_zstd.py b/Lib/test/test_zstd.py index 0763ba31737a4ca..5a7ffbdf054eb75 100644 --- a/Lib/test/test_zstd.py +++ b/Lib/test/test_zstd.py @@ -2958,5 +2958,49 @@ def run_method(b): pass +class OutputBufferTestCase(unittest.TestCase): + # Outputs at and around the boundaries of the internal output buffer's + # blocks (Include/internal/pycore_blocks_output_buffer.h): the first four + # (32 KiB, 64 KiB, 256 KiB and 1 MiB) are one buffer grown by realloc(); + # the next one (4 MiB) is a separate block on most platforms. + HEAD = (32 + 64 + 256 + 1024) * 1024 + SIZES = [HEAD - 1, HEAD, HEAD + 1, + HEAD + 4 * 1024 * 1024 - 1, HEAD + 4 * 1024 * 1024, + HEAD + 4 * 1024 * 1024 + 1] + # a pattern every codec compresses quickly: 256 varied bytes per 4 KiB + PATTERN = bytes(range(256)) + bytes(4096 - 256) + + @classmethod + def data(cls, size): + return (cls.PATTERN * (size // len(cls.PATTERN) + 1))[:size] + + @classmethod + def setUpClass(cls): + cls.compressed = {size: compress(cls.data(size), level=1) + for size in cls.SIZES} + + def test_decompress(self): + for size in self.SIZES: + with self.subTest(size=size): + data = self.data(size) + self.assertEqual(decompress(self.compressed[size]), data) + + def test_decompressor(self): + for size in self.SIZES: + with self.subTest(size=size): + data = self.data(size) + compressed = self.compressed[size] + self.assertEqual( + ZstdDecompressor().decompress(compressed), data) + # max_length past the first blocks and short of the next one + max_length = self.HEAD + 100 * 1024 + d = ZstdDecompressor() + first = d.decompress(compressed, max_length) + self.assertEqual(len(first), min(size, max_length)) + rest = b'' if d.eof else d.decompress(b'') + self.assertEqual(first + rest, data) + self.assertTrue(d.eof) + + if __name__ == "__main__": unittest.main() diff --git a/Misc/NEWS.d/next/Library/2026-11-05-21-00-00.gh-issue-161478.OutBuf.rst b/Misc/NEWS.d/next/Library/2026-11-05-21-00-00.gh-issue-161478.OutBuf.rst new file mode 100644 index 000000000000000..590130db5244974 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-11-05-21-00-00.gh-issue-161478.OutBuf.rst @@ -0,0 +1,4 @@ +Fix a performance regression on macOS and Windows: decompressing a few MiB +or more with :mod:`zlib`, :mod:`gzip`, :mod:`lzma` or :mod:`compression.zstd` +was up to three times slower than in 3.14, and on macOS the process kept +memory it no longer used.