diff --git a/Lib/encodings/base64_codec.py b/Lib/encodings/base64_codec.py index 8e7703b3b6072d6..ce5af465d6d9a4c 100644 --- a/Lib/encodings/base64_codec.py +++ b/Lib/encodings/base64_codec.py @@ -29,10 +29,25 @@ def encode(self, input, final=False): assert self.errors == 'strict' return base64.encodebytes(input) -class IncrementalDecoder(codecs.IncrementalDecoder): - def decode(self, input, final=False): - assert self.errors == 'strict' - return base64.decodebytes(input) +_alphabet = (b'ABCDEFGHIJKLMNOPQRSTUVWXYZ' + b'abcdefghijklmnopqrstuvwxyz0123456789+/') + +class IncrementalDecoder(codecs.BufferedIncrementalDecoder): + def _buffer_decode(self, input, errors, final): + assert errors == 'strict' + if not final: + # Only decode complete groups of four base64 characters and keep + # the rest for the next call. Other bytes, such as newlines and + # padding, are not counted. + count = len(input) - len(input.translate(None, _alphabet)) + remainder = count % 4 + end = len(input) + while remainder: + end -= 1 + if input[end] in _alphabet: + remainder -= 1 + input = input[:end] + return (base64.decodebytes(input), len(input)) class StreamWriter(Codec, codecs.StreamWriter): charbuffertype = bytes diff --git a/Lib/encodings/bz2_codec.py b/Lib/encodings/bz2_codec.py index fd9495e341baee1..a8fb0f3be919c27 100644 --- a/Lib/encodings/bz2_codec.py +++ b/Lib/encodings/bz2_codec.py @@ -52,7 +52,7 @@ def decode(self, input, final=False): try: return self.decompressobj.decompress(input) except EOFError: - return '' + return b'' def reset(self): self.decompressobj = bz2.BZ2Decompressor() diff --git a/Lib/test/test_codecs.py b/Lib/test/test_codecs.py index 37ac57063c35390..194e249f5adeed9 100644 --- a/Lib/test/test_codecs.py +++ b/Lib/test/test_codecs.py @@ -3251,6 +3251,34 @@ def test_uu_invalid(self): # Missing "begin" line self.assertRaises(ValueError, codecs.decode, b"", "uu-codec") + def test_base64_incremental_decode(self): + # gh-71986: Input split in the middle of a group of four characters + data = bytes(range(256)) + encoded = codecs.encode(data, "base64_codec") + for size in range(1, 6): + with self.subTest(size=size): + chunks = [encoded[i:i+size] + for i in range(0, len(encoded), size)] + decoded = b"".join(codecs.iterdecode(chunks, "base64_codec")) + self.assertEqual(decoded, data) + + decoded = codecs.iterdecode([b"AA", b"AA\r\n", b"A", b"A=="], + "base64_codec") + self.assertEqual(b"".join(decoded), b"\x00\x00\x00\x00") + + decoder = codecs.getincrementaldecoder("base64_codec")() + self.assertEqual(decoder.decode(b"AAA"), b"") + self.assertRaises(ValueError, decoder.decode, b"", final=True) + + @unittest.skipUnless("bz2_codec" in bytes_transform_encodings, + "Requires bz2 support") + def test_bz2_incremental_decode_after_end(self): + # gh-71986: The decoder returned str after the end of the stream + decoder = codecs.getincrementaldecoder("bz2_codec")() + self.assertEqual(decoder.decode(codecs.encode(b"data", "bz2_codec")), + b"data") + self.assertEqual(decoder.decode(b"", final=True), b"") + # The codec system tries to add notes to exceptions in order to ensure # the error mentions the operation being performed and the codec involved. diff --git a/Misc/NEWS.d/next/Library/2026-10-29-01-13-36.gh-issue-71986.iO8u7u.rst b/Misc/NEWS.d/next/Library/2026-10-29-01-13-36.gh-issue-71986.iO8u7u.rst new file mode 100644 index 000000000000000..779f031535c2f72 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-10-29-01-13-36.gh-issue-71986.iO8u7u.rst @@ -0,0 +1,5 @@ +Fix the incremental decoders of the ``base64_codec`` and ``bz2_codec`` +codecs. The ``base64_codec`` decoder now accepts input split in the middle +of a group of four characters, and the ``bz2_codec`` decoder now returns +:class:`bytes` instead of :class:`str` when called after the end of the +compressed data.