From 9a8714dc1cd8a247a73a1c4e1d0e6f23a7e40ba5 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Sun, 2 May 2021 06:49:03 -0700 Subject: [PATCH 001/129] bpo-36384: Leading zeros in IPv4 addresses are no longer tolerated (GH-25099) (GH-25815) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reverts commit e653d4d8e820a7a004ad399530af0135b45db27a and makes parsing even more strict. Like socket.inet_pton() any leading zero is now treated as invalid input. Signed-off-by: Christian Heimes Co-authored-by: Łukasz Langa (cherry picked from commit 60ce8f0be6354ad565393ab449d8de5d713f35bc) (cherry picked from commit 5374fbc31446364bf5f12e5ab88c5493c35eaf04) 3.7.17.x: 3.7 still had the octal/decimal ambiguity check that 3.8 removed (e653d4d8e82); it is now unreachable, so it and its test are dropped. --- Doc/library/ipaddress.rst | 19 +++++++++++-- Lib/ipaddress.py | 11 ++++---- Lib/test/test_ipaddress.py | 28 +++++++++++-------- .../2021-03-30-16-29-51.bpo-36384.sCAmLs.rst | 6 ++++ 4 files changed, 45 insertions(+), 19 deletions(-) create mode 100644 Misc/NEWS.d/next/Security/2021-03-30-16-29-51.bpo-36384.sCAmLs.rst diff --git a/Doc/library/ipaddress.rst b/Doc/library/ipaddress.rst index 1dfa2ec073313b8..22e2305abb2ab6f 100644 --- a/Doc/library/ipaddress.rst +++ b/Doc/library/ipaddress.rst @@ -104,8 +104,7 @@ write code that handles both IP versions correctly. Address objects are 1. A string in decimal-dot notation, consisting of four decimal integers in the inclusive range 0--255, separated by dots (e.g. ``192.168.0.1``). Each integer represents an octet (byte) in the address. Leading zeroes are - tolerated only for values less than 8 (as there is no ambiguity - between the decimal and octal interpretations of such strings). + not tolerated to prevent confusion with octal notation. 2. An integer that fits into 32 bits. 3. An integer packed into a :class:`bytes` object of length 4 (most significant octet first). @@ -117,6 +116,22 @@ write code that handles both IP versions correctly. Address objects are >>> ipaddress.IPv4Address(b'\xC0\xA8\x00\x01') IPv4Address('192.168.0.1') + .. versionchanged:: 3.8 + + Leading zeros are tolerated, even in ambiguous cases that look like + octal notation. + + .. versionchanged:: 3.10 + + Leading zeros are no longer tolerated and are treated as an error. + IPv4 address strings are now parsed as strict as glibc + :func:`~socket.inet_pton`. + + .. versionchanged:: 3.9.5 + + The above change was also included in Python 3.9 starting with + version 3.9.5. + .. attribute:: version The appropriate version number: ``4`` for IPv4, ``6`` for IPv6. diff --git a/Lib/ipaddress.py b/Lib/ipaddress.py index 7d1bbdcbc0819e5..07bb46f141d6e10 100644 --- a/Lib/ipaddress.py +++ b/Lib/ipaddress.py @@ -1190,14 +1190,13 @@ def _parse_octet(cls, octet_str): if len(octet_str) > 3: msg = "At most 3 characters permitted in %r" raise ValueError(msg % octet_str) + # Handle leading zeros as strict as glibc's inet_pton() + # See security bug bpo-36384 + if octet_str != '0' and octet_str[0] == '0': + msg = "Leading zeros are not permitted in %r" + raise ValueError(msg % octet_str) # Convert to integer (we know digits are legal) octet_int = int(octet_str, 10) - # Any octets that look like they *might* be written in octal, - # and which don't look exactly the same in both octal and - # decimal are rejected as ambiguous - if octet_int > 7 and octet_str[0] == '0': - msg = "Ambiguous (octal/decimal) value in %r not permitted" - raise ValueError(msg % octet_str) if octet_int > 255: raise ValueError("Octet %d (> 255) not permitted" % octet_int) return octet_int diff --git a/Lib/test/test_ipaddress.py b/Lib/test/test_ipaddress.py index 65b9168fe0ead76..19c2e1ca26d30b5 100644 --- a/Lib/test/test_ipaddress.py +++ b/Lib/test/test_ipaddress.py @@ -96,8 +96,23 @@ def pickle_test(self, addr): class CommonTestMixin_v4(CommonTestMixin): def test_leading_zeros(self): - self.assertInstancesEqual("000.000.000.000", "0.0.0.0") - self.assertInstancesEqual("192.168.000.001", "192.168.0.1") + # bpo-36384: no leading zeros to avoid ambiguity with octal notation + msg = "Leading zeros are not permitted in '\d+'" + addresses = [ + "000.000.000.000", + "192.168.000.001", + "016.016.016.016", + "192.168.000.001", + "001.000.008.016", + "01.2.3.40", + "1.02.3.40", + "1.2.03.40", + "1.2.3.040", + ] + for address in addresses: + with self.subTest(address=address): + with self.assertAddressError(msg): + self.factory(address) def test_int(self): self.assertInstancesEqual(0, "0.0.0.0") @@ -230,15 +245,6 @@ def assertBadOctet(addr, octet): assertBadOctet("1.2.3.4::", "4::") assertBadOctet("1.a.2.3", "a") - def test_octal_decimal_ambiguity(self): - def assertBadOctet(addr, octet): - msg = "Ambiguous (octal/decimal) value in %r not permitted in %r" - with self.assertAddressError(re.escape(msg % (octet, addr))): - ipaddress.IPv4Address(addr) - - assertBadOctet("016.016.016.016", "016") - assertBadOctet("001.000.008.016", "008") - def test_octet_length(self): def assertBadOctet(addr, octet): msg = "At most 3 characters permitted in %r in %r" diff --git a/Misc/NEWS.d/next/Security/2021-03-30-16-29-51.bpo-36384.sCAmLs.rst b/Misc/NEWS.d/next/Security/2021-03-30-16-29-51.bpo-36384.sCAmLs.rst new file mode 100644 index 000000000000000..f956cde948ec57d --- /dev/null +++ b/Misc/NEWS.d/next/Security/2021-03-30-16-29-51.bpo-36384.sCAmLs.rst @@ -0,0 +1,6 @@ +:mod:`ipaddress` module no longer accepts any leading zeros in IPv4 address +strings. Leading zeros are ambiguous and interpreted as octal notation by +some libraries. For example the legacy function :func:`socket.inet_aton` +treats leading zeros as octal notatation. glibc implementation of modern +:func:`~socket.inet_pton` does not accept any leading zeros. For a while +the :mod:`ipaddress` module used to accept ambiguous leading zeros. From d015d59f69ca9e5c4b30391802e7dafc7c779106 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Fri, 28 Oct 2022 03:08:30 -0700 Subject: [PATCH 002/129] [3.9] gh-97514: Don't use Linux abstract sockets for multiprocessing (GH-98501) (#98504) Linux abstract sockets are insecure as they lack any form of filesystem permissions so their use allows anyone on the system to inject code into the process. This removes the default preference for abstract sockets in multiprocessing introduced in Python 3.9+ via https://github.com/python/cpython/pull/18866 while fixing https://github.com/python/cpython/issues/84031. Explicit use of an abstract socket by a user now generates a RuntimeWarning. If we choose to keep this warning, it should be backported to the 3.7 and 3.8 branches. (cherry picked from commit 49f61068f49747164988ffc5a442d2a63874fc17) Co-authored-by: Gregory P. Smith (cherry picked from commit b43496c01a554cf41ae654a0379efae18609ad39) --- .../2022-09-07-10-42-00.gh-issue-97514.Yggdsl.rst | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 Misc/NEWS.d/next/Security/2022-09-07-10-42-00.gh-issue-97514.Yggdsl.rst diff --git a/Misc/NEWS.d/next/Security/2022-09-07-10-42-00.gh-issue-97514.Yggdsl.rst b/Misc/NEWS.d/next/Security/2022-09-07-10-42-00.gh-issue-97514.Yggdsl.rst new file mode 100644 index 000000000000000..02d95b570520dff --- /dev/null +++ b/Misc/NEWS.d/next/Security/2022-09-07-10-42-00.gh-issue-97514.Yggdsl.rst @@ -0,0 +1,15 @@ +On Linux the :mod:`multiprocessing` module returns to using filesystem backed +unix domain sockets for communication with the *forkserver* process instead of +the Linux abstract socket namespace. Only code that chooses to use the +:ref:`"forkserver" start method ` is affected. + +Abstract sockets have no permissions and could allow any user on the system in +the same `network namespace +`_ (often the +whole system) to inject code into the multiprocessing *forkserver* process. +This was a potential privilege escalation. Filesystem based socket permissions +restrict this to the *forkserver* process user as was the default in Python 3.8 +and earlier. + +This prevents Linux `CVE-2022-42919 +`_. From f75758eb3fe48cfd70b4d4b9f250b0fca646eeeb Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Tue, 22 Aug 2023 20:28:10 +0200 Subject: [PATCH 003/129] [3.9] gh-107845: Fix symlink handling for tarfile.data_filter (GH-107846) (#108274) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit (cherry picked from commit acbd3f9c5c5f23e95267714e41236140d84fe962) Co-authored-by: Petr Viktorin Co-authored-by: Victor Stinner Co-authored-by: Lumír 'Frenzy' Balhar (cherry picked from commit 42deeab5b2efc2930d4eb73416e1dde9cf790dd2) --- Doc/library/tarfile.rst | 5 + Lib/tarfile.py | 11 +- Lib/test/test_tarfile.py | 144 +++++++++++++++++- ...-08-10-17-36-22.gh-issue-107845.dABiMJ.rst | 3 + 4 files changed, 154 insertions(+), 9 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2023-08-10-17-36-22.gh-issue-107845.dABiMJ.rst diff --git a/Doc/library/tarfile.rst b/Doc/library/tarfile.rst index 2610fe3a683f01d..9c37df44beec204 100644 --- a/Doc/library/tarfile.rst +++ b/Doc/library/tarfile.rst @@ -713,6 +713,11 @@ A ``TarInfo`` object has the following public data attributes: Name of the target file name, which is only present in :class:`TarInfo` objects of type :const:`LNKTYPE` and :const:`SYMTYPE`. + For symbolic links (``SYMTYPE``), the *linkname* is relative to the directory + that contains the link. + For hard links (``LNKTYPE``), the *linkname* is relative to the root of + the archive. + .. attribute:: TarInfo.uid diff --git a/Lib/tarfile.py b/Lib/tarfile.py index 2731c379722a455..41cfd6f2c62119a 100755 --- a/Lib/tarfile.py +++ b/Lib/tarfile.py @@ -750,7 +750,7 @@ def __init__(self, tarinfo): class AbsoluteLinkError(FilterError): def __init__(self, tarinfo): self.tarinfo = tarinfo - super().__init__(f'{tarinfo.name!r} is a symlink to an absolute path') + super().__init__(f'{tarinfo.name!r} is a link to an absolute path') class LinkOutsideDestinationError(FilterError): def __init__(self, tarinfo, path): @@ -810,7 +810,14 @@ def _get_filtered_attrs(member, dest_path, for_data=True): if member.islnk() or member.issym(): if os.path.isabs(member.linkname): raise AbsoluteLinkError(member) - target_path = os.path.realpath(os.path.join(dest_path, member.linkname)) + if member.issym(): + target_path = os.path.join(dest_path, + os.path.dirname(name), + member.linkname) + else: + target_path = os.path.join(dest_path, + member.linkname) + target_path = os.path.realpath(target_path) if os.path.commonpath([target_path, dest_path]) != dest_path: raise LinkOutsideDestinationError(member, target_path) return new_attrs diff --git a/Lib/test/test_tarfile.py b/Lib/test/test_tarfile.py index 64ab69796d1d386..13eec2f276a3081 100644 --- a/Lib/test/test_tarfile.py +++ b/Lib/test/test_tarfile.py @@ -3026,10 +3026,12 @@ def __exit__(self, *exc): self.bio = None def add(self, name, *, type=None, symlink_to=None, hardlink_to=None, - mode=None, **kwargs): + mode=None, size=None, **kwargs): """Add a member to the test archive. Call within `with`.""" name = str(name) tarinfo = tarfile.TarInfo(name).replace(**kwargs) + if size is not None: + tarinfo.size = size if mode: tarinfo.mode = _filemode_to_int(mode) if symlink_to is not None: @@ -3102,7 +3104,8 @@ def check_context(self, tar, filter): raise self.raised_exception self.assertEqual(self.expected_paths, set()) - def expect_file(self, name, type=None, symlink_to=None, mode=None): + def expect_file(self, name, type=None, symlink_to=None, mode=None, + size=None): """Check a single file. See check_context.""" if self.raised_exception: raise self.raised_exception @@ -3136,6 +3139,8 @@ def expect_file(self, name, type=None, symlink_to=None, mode=None): self.assertTrue(path.is_fifo()) else: raise NotImplementedError(type) + if size is not None: + self.assertEqual(path.stat().st_size, size) for parent in path.parents: self.expected_paths.discard(parent) @@ -3181,8 +3186,15 @@ def test_parent_symlink(self): # Test interplaying symlinks # Inspired by 'dirsymlink2a' in jwilk/traversal-archives with ArchiveMaker() as arc: + + # `current` links to `.` which is both: + # - the destination directory + # - `current` itself arc.add('current', symlink_to='.') + + # effectively points to ./../ arc.add('parent', symlink_to='current/..') + arc.add('parent/evil') if support.can_symlink(): @@ -3223,9 +3235,46 @@ def test_parent_symlink(self): def test_parent_symlink2(self): # Test interplaying symlinks # Inspired by 'dirsymlink2b' in jwilk/traversal-archives + + # Posix and Windows have different pathname resolution: + # either symlink or a '..' component resolve first. + # Let's see which we are on. + if support.can_symlink(): + testpath = os.path.join(TEMPDIR, 'resolution_test') + os.mkdir(testpath) + + # testpath/current links to `.` which is all of: + # - `testpath` + # - `testpath/current` + # - `testpath/current/current` + # - etc. + os.symlink('.', os.path.join(testpath, 'current')) + + # we'll test where `testpath/current/../file` ends up + with open(os.path.join(testpath, 'current', '..', 'file'), 'w'): + pass + + if os.path.exists(os.path.join(testpath, 'file')): + # Windows collapses 'current\..' to '.' first, leaving + # 'testpath\file' + dotdot_resolves_early = True + elif os.path.exists(os.path.join(testpath, '..', 'file')): + # Posix resolves 'current' to '.' first, leaving + # 'testpath/../file' + dotdot_resolves_early = False + else: + raise AssertionError('Could not determine link resolution') + with ArchiveMaker() as arc: + + # `current` links to `.` which is both the destination directory + # and `current` itself arc.add('current', symlink_to='.') + + # `current/parent` is also available as `./parent`, + # and effectively points to `./../` arc.add('current/parent', symlink_to='..') + arc.add('parent/evil') with self.check_context(arc.open(), 'fully_trusted'): @@ -3239,6 +3288,7 @@ def test_parent_symlink2(self): with self.check_context(arc.open(), 'tar'): if support.can_symlink(): + # Fail when extracting a file outside destination self.expect_exception( tarfile.OutsideDestinationError, "'parent/evil' would be extracted to " @@ -3249,10 +3299,24 @@ def test_parent_symlink2(self): self.expect_file('parent/evil') with self.check_context(arc.open(), 'data'): - self.expect_exception( - tarfile.LinkOutsideDestinationError, - """'current/parent' would link to ['"].*['"], """ - + "which is outside the destination") + if support.can_symlink(): + if dotdot_resolves_early: + # Fail when extracting a file outside destination + self.expect_exception( + tarfile.OutsideDestinationError, + "'parent/evil' would be extracted to " + + """['"].*evil['"], which is outside """ + + "the destination") + else: + # Fail as soon as we have a symlink outside the destination + self.expect_exception( + tarfile.LinkOutsideDestinationError, + "'current/parent' would link to " + + """['"].*outerdir['"], which is outside """ + + "the destination") + else: + self.expect_file('current/') + self.expect_file('parent/evil') def test_absolute_symlink(self): # Test symlink to an absolute path @@ -3281,11 +3345,29 @@ def test_absolute_symlink(self): with self.check_context(arc.open(), 'data'): self.expect_exception( tarfile.AbsoluteLinkError, - "'parent' is a symlink to an absolute path") + "'parent' is a link to an absolute path") + + def test_absolute_hardlink(self): + # Test hardlink to an absolute path + # Inspired by 'dirsymlink' in https://github.com/jwilk/traversal-archives + with ArchiveMaker() as arc: + arc.add('parent', hardlink_to=self.outerdir / 'foo') + + with self.check_context(arc.open(), 'fully_trusted'): + self.expect_exception(KeyError, ".*foo. not found") + + with self.check_context(arc.open(), 'tar'): + self.expect_exception(KeyError, ".*foo. not found") + + with self.check_context(arc.open(), 'data'): + self.expect_exception( + tarfile.AbsoluteLinkError, + "'parent' is a link to an absolute path") def test_sly_relative0(self): # Inspired by 'relative0' in jwilk/traversal-archives with ArchiveMaker() as arc: + # points to `../../tmp/moo` arc.add('../moo', symlink_to='..//tmp/moo') try: @@ -3335,6 +3417,54 @@ def test_sly_relative2(self): + """['"].*moo['"], which is outside the """ + "destination") + def test_deep_symlink(self): + # Test that symlinks and hardlinks inside a directory + # point to the correct file (`target` of size 3). + # If links aren't supported we get a copy of the file. + with ArchiveMaker() as arc: + arc.add('targetdir/target', size=3) + # a hardlink's linkname is relative to the archive + arc.add('linkdir/hardlink', hardlink_to=os.path.join( + 'targetdir', 'target')) + # a symlink's linkname is relative to the link's directory + arc.add('linkdir/symlink', symlink_to=os.path.join( + '..', 'targetdir', 'target')) + + for filter in 'tar', 'data', 'fully_trusted': + with self.check_context(arc.open(), filter): + self.expect_file('targetdir/target', size=3) + self.expect_file('linkdir/hardlink', size=3) + if support.can_symlink(): + self.expect_file('linkdir/symlink', size=3, + symlink_to='../targetdir/target') + else: + self.expect_file('linkdir/symlink', size=3) + + def test_chains(self): + # Test chaining of symlinks/hardlinks. + # Symlinks are created before the files they point to. + with ArchiveMaker() as arc: + arc.add('linkdir/symlink', symlink_to='hardlink') + arc.add('symlink2', symlink_to=os.path.join( + 'linkdir', 'hardlink2')) + arc.add('targetdir/target', size=3) + arc.add('linkdir/hardlink', hardlink_to='targetdir/target') + arc.add('linkdir/hardlink2', hardlink_to='linkdir/symlink') + + for filter in 'tar', 'data', 'fully_trusted': + with self.check_context(arc.open(), filter): + self.expect_file('targetdir/target', size=3) + self.expect_file('linkdir/hardlink', size=3) + self.expect_file('linkdir/hardlink2', size=3) + if support.can_symlink(): + self.expect_file('linkdir/symlink', size=3, + symlink_to='hardlink') + self.expect_file('symlink2', size=3, + symlink_to='linkdir/hardlink2') + else: + self.expect_file('linkdir/symlink', size=3) + self.expect_file('symlink2', size=3) + def test_modes(self): # Test how file modes are extracted # (Note that the modes are ignored on platforms without working chmod) diff --git a/Misc/NEWS.d/next/Library/2023-08-10-17-36-22.gh-issue-107845.dABiMJ.rst b/Misc/NEWS.d/next/Library/2023-08-10-17-36-22.gh-issue-107845.dABiMJ.rst new file mode 100644 index 000000000000000..32c1fb93f4ab2ce --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-08-10-17-36-22.gh-issue-107845.dABiMJ.rst @@ -0,0 +1,3 @@ +:func:`tarfile.data_filter` now takes the location of symlinks into account +when determining their target, so it will no longer reject some valid +tarballs with ``LinkOutsideDestinationError``. From d4efb307eff4f3aa84bcdf67fffe4bc548457d81 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Langa?= Date: Mon, 6 Nov 2023 18:44:50 +0100 Subject: [PATCH 004/129] [3.9] gh-101180: Fix a bug where iso2022_jp_3 and iso2022_jp_2004 codecs read out of bounds (gh-111695) (gh-111780) (cherry picked from commit c8faa3568afd255708096f6aa8df0afa80cf7697) Co-authored-by: Masayuki Moriyama (cherry picked from commit 08b640e1575b75aeb605e72cb30c337480055b75) --- Lib/test/test_codecencodings_iso2022.py | 46 +++++++++++++++++++ ...-10-27-19-38-33.gh-issue-102388.vd5YUZ.rst | 1 + Modules/cjkcodecs/_codecs_iso2022.c | 9 ++-- 3 files changed, 53 insertions(+), 3 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-10-27-19-38-33.gh-issue-102388.vd5YUZ.rst diff --git a/Lib/test/test_codecencodings_iso2022.py b/Lib/test/test_codecencodings_iso2022.py index 00ea1c39dd6fb66..027dbecc6134df7 100644 --- a/Lib/test/test_codecencodings_iso2022.py +++ b/Lib/test/test_codecencodings_iso2022.py @@ -24,6 +24,52 @@ class Test_ISO2022_JP2(multibytecodec_support.TestBase, unittest.TestCase): (b'ab\x1BNdef', 'replace', 'abdef'), ) +class Test_ISO2022_JP3(multibytecodec_support.TestBase, unittest.TestCase): + encoding = 'iso2022_jp_3' + tstring = multibytecodec_support.load_teststring('iso2022_jp') + codectests = COMMON_CODEC_TESTS + ( + (b'ab\x1BNdef', 'replace', 'ab\x1BNdef'), + (b'\x1B$(O\x2E\x23\x1B(B', 'strict', '\u3402' ), + (b'\x1B$(O\x2E\x22\x1B(B', 'strict', '\U0002000B' ), + (b'\x1B$(O\x24\x77\x1B(B', 'strict', '\u304B\u309A'), + (b'\x1B$(P\x21\x22\x1B(B', 'strict', '\u4E02' ), + (b'\x1B$(P\x7E\x76\x1B(B', 'strict', '\U0002A6B2' ), + ('\u3402', 'strict', b'\x1B$(O\x2E\x23\x1B(B'), + ('\U0002000B', 'strict', b'\x1B$(O\x2E\x22\x1B(B'), + ('\u304B\u309A', 'strict', b'\x1B$(O\x24\x77\x1B(B'), + ('\u4E02', 'strict', b'\x1B$(P\x21\x22\x1B(B'), + ('\U0002A6B2', 'strict', b'\x1B$(P\x7E\x76\x1B(B'), + (b'ab\x1B$(O\x2E\x21\x1B(Bdef', 'replace', 'ab\uFFFDdef'), + ('ab\u4FF1def', 'replace', b'ab?def'), + ) + xmlcharnametest = ( + '\xAB\u211C\xBB = \u2329\u1234\u232A', + b'\x1B$(O\x29\x28\x1B(Bℜ\x1B$(O\x29\x32\x1B(B = ⟨ሴ⟩' + ) + +class Test_ISO2022_JP2004(multibytecodec_support.TestBase, unittest.TestCase): + encoding = 'iso2022_jp_2004' + tstring = multibytecodec_support.load_teststring('iso2022_jp') + codectests = COMMON_CODEC_TESTS + ( + (b'ab\x1BNdef', 'replace', 'ab\x1BNdef'), + (b'\x1B$(Q\x2E\x23\x1B(B', 'strict', '\u3402' ), + (b'\x1B$(Q\x2E\x22\x1B(B', 'strict', '\U0002000B' ), + (b'\x1B$(Q\x24\x77\x1B(B', 'strict', '\u304B\u309A'), + (b'\x1B$(P\x21\x22\x1B(B', 'strict', '\u4E02' ), + (b'\x1B$(P\x7E\x76\x1B(B', 'strict', '\U0002A6B2' ), + ('\u3402', 'strict', b'\x1B$(Q\x2E\x23\x1B(B'), + ('\U0002000B', 'strict', b'\x1B$(Q\x2E\x22\x1B(B'), + ('\u304B\u309A', 'strict', b'\x1B$(Q\x24\x77\x1B(B'), + ('\u4E02', 'strict', b'\x1B$(P\x21\x22\x1B(B'), + ('\U0002A6B2', 'strict', b'\x1B$(P\x7E\x76\x1B(B'), + (b'ab\x1B$(Q\x2E\x21\x1B(Bdef', 'replace', 'ab\u4FF1def'), + ('ab\u4FF1def', 'replace', b'ab\x1B$(Q\x2E\x21\x1B(Bdef'), + ) + xmlcharnametest = ( + '\xAB\u211C\xBB = \u2329\u1234\u232A', + b'\x1B$(Q\x29\x28\x1B(Bℜ\x1B$(Q\x29\x32\x1B(B = ⟨ሴ⟩' + ) + class Test_ISO2022_KR(multibytecodec_support.TestBase, unittest.TestCase): encoding = 'iso2022_kr' tstring = multibytecodec_support.load_teststring('iso2022_kr') diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-10-27-19-38-33.gh-issue-102388.vd5YUZ.rst b/Misc/NEWS.d/next/Core and Builtins/2023-10-27-19-38-33.gh-issue-102388.vd5YUZ.rst new file mode 100644 index 000000000000000..268a3d310f2b499 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-10-27-19-38-33.gh-issue-102388.vd5YUZ.rst @@ -0,0 +1 @@ +Fix a bug where ``iso2022_jp_3`` and ``iso2022_jp_2004`` codecs read out of bounds diff --git a/Modules/cjkcodecs/_codecs_iso2022.c b/Modules/cjkcodecs/_codecs_iso2022.c index 7394cf67e0e7dd5..6d906ecdd396c29 100644 --- a/Modules/cjkcodecs/_codecs_iso2022.c +++ b/Modules/cjkcodecs/_codecs_iso2022.c @@ -181,8 +181,9 @@ ENCODER(iso2022) encoded = MAP_UNMAPPABLE; for (dsg = CONFIG_DESIGNATIONS; dsg->mark; dsg++) { + Py_UCS4 buf[2] = {c, 0}; Py_ssize_t length = 1; - encoded = dsg->encoder(&c, &length); + encoded = dsg->encoder(buf, &length); if (encoded == MAP_MULTIPLE_AVAIL) { /* this implementation won't work for pair * of non-bmp characters. */ @@ -191,9 +192,11 @@ ENCODER(iso2022) return MBERR_TOOFEW; length = -1; } - else + else { + buf[1] = INCHAR2; length = 2; - encoded = dsg->encoder(&c, &length); + } + encoded = dsg->encoder(buf, &length); if (encoded != MAP_UNMAPPABLE) { insize = length; break; From bed6f1e48aaf05ee63936d330346ade82b4fafba Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Wed, 17 Jan 2024 15:47:47 +0200 Subject: [PATCH 005/129] [3.9] gh-91133: tempfile.TemporaryDirectory: fix symlink bug in cleanup (GH-99930) (GH-112842) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit (cherry picked from commit 81c16cd94ec38d61aa478b9a452436dc3b1b524d) Co-authored-by: Søren Løvborg (cherry picked from commit d54e22a669ae6e987199bb5d2c69bb5a46b0083b) 3.7.17.x: the TemporaryDirectory test_modes/test_flags tests come along (3.7 already had the chmod/chflags cleanup they cover); test imports stat and uses test.support.skip_unless_symlink. --- Lib/tempfile.py | 27 ++-- Lib/test/test_tempfile.py | 145 +++++++++++++++++- ...2-12-01-16-57-44.gh-issue-91133.LKMVCV.rst | 2 + 3 files changed, 164 insertions(+), 10 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2022-12-01-16-57-44.gh-issue-91133.LKMVCV.rst diff --git a/Lib/tempfile.py b/Lib/tempfile.py index 3cb53a815b4b116..68d8d4c868bcf13 100644 --- a/Lib/tempfile.py +++ b/Lib/tempfile.py @@ -271,6 +271,22 @@ def _mkstemp_inner(dir, pre, suf, flags, output_type): raise FileExistsError(_errno.EEXIST, "No usable temporary file name found") +def _dont_follow_symlinks(func, path, *args): + # Pass follow_symlinks=False, unless not supported on this platform. + if func in _os.supports_follow_symlinks: + func(path, *args, follow_symlinks=False) + elif _os.name == 'nt' or not _os.path.islink(path): + func(path, *args) + +def _resetperms(path): + try: + chflags = _os.chflags + except AttributeError: + pass + else: + _dont_follow_symlinks(chflags, path, 0) + _dont_follow_symlinks(_os.chmod, path, 0o700) + # User visible interfaces. @@ -799,17 +815,10 @@ def __init__(self, suffix=None, prefix=None, dir=None, def _rmtree(cls, name, ignore_errors=False): def onerror(func, path, exc_info): if issubclass(exc_info[0], PermissionError): - def resetperms(path): - try: - _os.chflags(path, 0) - except AttributeError: - pass - _os.chmod(path, 0o700) - try: if path != name: - resetperms(_os.path.dirname(path)) - resetperms(path) + _resetperms(_os.path.dirname(path)) + _resetperms(path) try: _os.unlink(path) diff --git a/Lib/test/test_tempfile.py b/Lib/test/test_tempfile.py index 8171aa6e0ae52af..63f1412629c366e 100644 --- a/Lib/test/test_tempfile.py +++ b/Lib/test/test_tempfile.py @@ -5,6 +5,7 @@ import os import pathlib import signal +import stat import sys import re import warnings @@ -1379,7 +1380,7 @@ def test_explict_cleanup_ignore_errors(self): temp_path.exists(), f"TemporaryDirectory {temp_path!s} exists after cleanup") - @os_helper.skip_unless_symlink + @support.skip_unless_symlink def test_cleanup_with_symlink_to_a_directory(self): # cleanup() should not follow symlinks to directories (issue #12464) d1 = self.do_create() @@ -1400,6 +1401,103 @@ def test_cleanup_with_symlink_to_a_directory(self): "were deleted") d2.cleanup() + @support.skip_unless_symlink + def test_cleanup_with_symlink_modes(self): + # cleanup() should not follow symlinks when fixing mode bits (#91133) + with self.do_create(recurse=0) as d2: + file1 = os.path.join(d2, 'file1') + open(file1, 'wb').close() + dir1 = os.path.join(d2, 'dir1') + os.mkdir(dir1) + for mode in range(8): + mode <<= 6 + with self.subTest(mode=format(mode, '03o')): + def test(target, target_is_directory): + d1 = self.do_create(recurse=0) + symlink = os.path.join(d1.name, 'symlink') + os.symlink(target, symlink, + target_is_directory=target_is_directory) + try: + os.chmod(symlink, mode, follow_symlinks=False) + except NotImplementedError: + pass + try: + os.chmod(symlink, mode) + except FileNotFoundError: + pass + os.chmod(d1.name, mode) + d1.cleanup() + self.assertFalse(os.path.exists(d1.name)) + + with self.subTest('nonexisting file'): + test('nonexisting', target_is_directory=False) + with self.subTest('nonexisting dir'): + test('nonexisting', target_is_directory=True) + + with self.subTest('existing file'): + os.chmod(file1, mode) + old_mode = os.stat(file1).st_mode + test(file1, target_is_directory=False) + new_mode = os.stat(file1).st_mode + self.assertEqual(new_mode, old_mode, + '%03o != %03o' % (new_mode, old_mode)) + + with self.subTest('existing dir'): + os.chmod(dir1, mode) + old_mode = os.stat(dir1).st_mode + test(dir1, target_is_directory=True) + new_mode = os.stat(dir1).st_mode + self.assertEqual(new_mode, old_mode, + '%03o != %03o' % (new_mode, old_mode)) + + @unittest.skipUnless(hasattr(os, 'chflags'), 'requires os.chflags') + @support.skip_unless_symlink + def test_cleanup_with_symlink_flags(self): + # cleanup() should not follow symlinks when fixing flags (#91133) + flags = stat.UF_IMMUTABLE | stat.UF_NOUNLINK + self.check_flags(flags) + + with self.do_create(recurse=0) as d2: + file1 = os.path.join(d2, 'file1') + open(file1, 'wb').close() + dir1 = os.path.join(d2, 'dir1') + os.mkdir(dir1) + def test(target, target_is_directory): + d1 = self.do_create(recurse=0) + symlink = os.path.join(d1.name, 'symlink') + os.symlink(target, symlink, + target_is_directory=target_is_directory) + try: + os.chflags(symlink, flags, follow_symlinks=False) + except NotImplementedError: + pass + try: + os.chflags(symlink, flags) + except FileNotFoundError: + pass + os.chflags(d1.name, flags) + d1.cleanup() + self.assertFalse(os.path.exists(d1.name)) + + with self.subTest('nonexisting file'): + test('nonexisting', target_is_directory=False) + with self.subTest('nonexisting dir'): + test('nonexisting', target_is_directory=True) + + with self.subTest('existing file'): + os.chflags(file1, flags) + old_flags = os.stat(file1).st_flags + test(file1, target_is_directory=False) + new_flags = os.stat(file1).st_flags + self.assertEqual(new_flags, old_flags) + + with self.subTest('existing dir'): + os.chflags(dir1, flags) + old_flags = os.stat(dir1).st_flags + test(dir1, target_is_directory=True) + new_flags = os.stat(dir1).st_flags + self.assertEqual(new_flags, old_flags) + @support.cpython_only def test_del_on_collection(self): # A TemporaryDirectory is deleted when garbage collected @@ -1556,6 +1654,51 @@ def test_context_manager(self): self.assertEqual(name, d.name) self.assertFalse(os.path.exists(name)) + def test_modes(self): + for mode in range(8): + mode <<= 6 + with self.subTest(mode=format(mode, '03o')): + d = self.do_create(recurse=3, dirs=2, files=2) + with d: + # Change files and directories mode recursively. + for root, dirs, files in os.walk(d.name, topdown=False): + for name in files: + os.chmod(os.path.join(root, name), mode) + os.chmod(root, mode) + d.cleanup() + self.assertFalse(os.path.exists(d.name)) + + def check_flags(self, flags): + # skip the test if these flags are not supported (ex: FreeBSD 13) + filename = support.TESTFN + try: + open(filename, "w").close() + try: + os.chflags(filename, flags) + except OSError as exc: + # "OSError: [Errno 45] Operation not supported" + self.skipTest(f"chflags() doesn't support flags " + f"{flags:#b}: {exc}") + else: + os.chflags(filename, 0) + finally: + support.unlink(filename) + + @unittest.skipUnless(hasattr(os, 'chflags'), 'requires os.chflags') + def test_flags(self): + flags = stat.UF_IMMUTABLE | stat.UF_NOUNLINK + self.check_flags(flags) + + d = self.do_create(recurse=3, dirs=2, files=2) + with d: + # Change files and directories flags recursively. + for root, dirs, files in os.walk(d.name, topdown=False): + for name in files: + os.chflags(os.path.join(root, name), flags) + os.chflags(root, flags) + d.cleanup() + self.assertFalse(os.path.exists(d.name)) + if __name__ == "__main__": unittest.main() diff --git a/Misc/NEWS.d/next/Library/2022-12-01-16-57-44.gh-issue-91133.LKMVCV.rst b/Misc/NEWS.d/next/Library/2022-12-01-16-57-44.gh-issue-91133.LKMVCV.rst new file mode 100644 index 000000000000000..7991048fc48e039 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-12-01-16-57-44.gh-issue-91133.LKMVCV.rst @@ -0,0 +1,2 @@ +Fix a bug in :class:`tempfile.TemporaryDirectory` cleanup, which now no longer +dereferences symlinks when working around file system permission errors. From c35e547f3e6133ab186d767bba67c766bbea3e71 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Wed, 17 Jan 2024 14:48:06 +0100 Subject: [PATCH 006/129] [3.9] gh-109858: Protect zipfile from "quoted-overlap" zipbomb (GH-110016) (GH-113915) Raise BadZipFile when try to read an entry that overlaps with other entry or central directory. (cherry picked from commit 66363b9a7b9fe7c99eba3a185b74c5fdbf842eba) Co-authored-by: Serhiy Storchaka (cherry picked from commit a2c59992e9e8d35baba9695eb186ad6c6ff85c51) 3.7.17.x: ZipInfo.__init__ only gains _end_offset (3.7 never initialized compress_size/file_size there); tests use the 3.7 requires_zlib decorator; the unrelated bz2 test in the hunk is dropped. --- Lib/test/test_zipfile.py | 60 +++++++++++++++++++ Lib/zipfile.py | 12 ++++ ...-09-28-13-15-51.gh-issue-109858.43e2dg.rst | 3 + 3 files changed, 75 insertions(+) create mode 100644 Misc/NEWS.d/next/Library/2023-09-28-13-15-51.gh-issue-109858.43e2dg.rst diff --git a/Lib/test/test_zipfile.py b/Lib/test/test_zipfile.py index 6a6df859856cdd8..16bfb4a80907d18 100644 --- a/Lib/test/test_zipfile.py +++ b/Lib/test/test_zipfile.py @@ -1939,6 +1939,66 @@ def test_seek_tell(self): fp.seek(0, os.SEEK_SET) self.assertEqual(fp.tell(), 0) + @requires_zlib + def test_full_overlap(self): + data = ( + b'PK\x03\x04\x14\x00\x00\x00\x08\x00\xa0lH\x05\xe2\x1e' + b'8\xbb\x10\x00\x00\x00\t\x04\x00\x00\x01\x00\x00\x00a\xed' + b'\xc0\x81\x08\x00\x00\x00\xc00\xd6\xfbK\\d\x0b`P' + b'K\x01\x02\x14\x00\x14\x00\x00\x00\x08\x00\xa0lH\x05\xe2' + b'\x1e8\xbb\x10\x00\x00\x00\t\x04\x00\x00\x01\x00\x00\x00\x00' + b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00aPK' + b'\x01\x02\x14\x00\x14\x00\x00\x00\x08\x00\xa0lH\x05\xe2\x1e' + b'8\xbb\x10\x00\x00\x00\t\x04\x00\x00\x01\x00\x00\x00\x00\x00' + b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00bPK\x05' + b'\x06\x00\x00\x00\x00\x02\x00\x02\x00^\x00\x00\x00/\x00\x00' + b'\x00\x00\x00' + ) + with zipfile.ZipFile(io.BytesIO(data), 'r') as zipf: + self.assertEqual(zipf.namelist(), ['a', 'b']) + zi = zipf.getinfo('a') + self.assertEqual(zi.header_offset, 0) + self.assertEqual(zi.compress_size, 16) + self.assertEqual(zi.file_size, 1033) + zi = zipf.getinfo('b') + self.assertEqual(zi.header_offset, 0) + self.assertEqual(zi.compress_size, 16) + self.assertEqual(zi.file_size, 1033) + self.assertEqual(len(zipf.read('a')), 1033) + with self.assertRaisesRegex(zipfile.BadZipFile, 'File name.*differ'): + zipf.read('b') + + @requires_zlib + def test_quoted_overlap(self): + data = ( + b'PK\x03\x04\x14\x00\x00\x00\x08\x00\xa0lH\x05Y\xfc' + b'8\x044\x00\x00\x00(\x04\x00\x00\x01\x00\x00\x00a\x00' + b'\x1f\x00\xe0\xffPK\x03\x04\x14\x00\x00\x00\x08\x00\xa0l' + b'H\x05\xe2\x1e8\xbb\x10\x00\x00\x00\t\x04\x00\x00\x01\x00' + b'\x00\x00b\xed\xc0\x81\x08\x00\x00\x00\xc00\xd6\xfbK\\' + b'd\x0b`PK\x01\x02\x14\x00\x14\x00\x00\x00\x08\x00\xa0' + b'lH\x05Y\xfc8\x044\x00\x00\x00(\x04\x00\x00\x01' + b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00' + b'\x00aPK\x01\x02\x14\x00\x14\x00\x00\x00\x08\x00\xa0l' + b'H\x05\xe2\x1e8\xbb\x10\x00\x00\x00\t\x04\x00\x00\x01\x00' + b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00$\x00\x00\x00' + b'bPK\x05\x06\x00\x00\x00\x00\x02\x00\x02\x00^\x00\x00' + b'\x00S\x00\x00\x00\x00\x00' + ) + with zipfile.ZipFile(io.BytesIO(data), 'r') as zipf: + self.assertEqual(zipf.namelist(), ['a', 'b']) + zi = zipf.getinfo('a') + self.assertEqual(zi.header_offset, 0) + self.assertEqual(zi.compress_size, 52) + self.assertEqual(zi.file_size, 1064) + zi = zipf.getinfo('b') + self.assertEqual(zi.header_offset, 36) + self.assertEqual(zi.compress_size, 16) + self.assertEqual(zi.file_size, 1033) + with self.assertRaisesRegex(zipfile.BadZipFile, 'Overlapped entries'): + zipf.read('a') + self.assertEqual(len(zipf.read('b')), 1033) + def tearDown(self): unlink(TESTFN) unlink(TESTFN2) diff --git a/Lib/zipfile.py b/Lib/zipfile.py index 63748d371f0e755..8eefe60a7bdb955 100644 --- a/Lib/zipfile.py +++ b/Lib/zipfile.py @@ -334,6 +334,7 @@ class ZipInfo (object): 'compress_size', 'file_size', '_raw_time', + '_end_offset', ) def __init__(self, filename="NoName", date_time=(1980,1,1,0,0,0)): @@ -373,6 +374,7 @@ def __init__(self, filename="NoName", date_time=(1980,1,1,0,0,0)): self.volume = 0 # Volume number of file header self.internal_attr = 0 # Internal attributes self.external_attr = 0 # External file attributes + self._end_offset = None # Start of the next local header or central directory # Other attributes are set by class ZipFile: # header_offset Byte offset to the file header # CRC CRC-32 of the uncompressed file @@ -1391,6 +1393,12 @@ def _RealGetContents(self): if self.debug > 2: print("total", total) + end_offset = self.start_dir + for zinfo in sorted(self.filelist, + key=lambda zinfo: zinfo.header_offset, + reverse=True): + zinfo._end_offset = end_offset + end_offset = zinfo.header_offset def namelist(self): """Return a list of file names in the archive.""" @@ -1546,6 +1554,10 @@ def open(self, name, mode="r", pwd=None, *, force_zip64=False): 'File name in directory %r and header %r differ.' % (zinfo.orig_filename, fname)) + if (zinfo._end_offset is not None and + zef_file.tell() + zinfo.compress_size > zinfo._end_offset): + raise BadZipFile(f"Overlapped entries: {zinfo.orig_filename!r} (possible zip bomb)") + # check for encrypted flag & handle password is_encrypted = zinfo.flag_bits & 0x1 if is_encrypted: diff --git a/Misc/NEWS.d/next/Library/2023-09-28-13-15-51.gh-issue-109858.43e2dg.rst b/Misc/NEWS.d/next/Library/2023-09-28-13-15-51.gh-issue-109858.43e2dg.rst new file mode 100644 index 000000000000000..be279caffc46eeb --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-09-28-13-15-51.gh-issue-109858.43e2dg.rst @@ -0,0 +1,3 @@ +Protect :mod:`zipfile` from "quoted-overlap" zipbomb. It now raises +BadZipFile when try to read an entry that overlaps with other entry or +central directory. From 793ad2489439336e52ed3479d2a4084faf0f8c81 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Wed, 17 Jan 2024 16:28:17 +0200 Subject: [PATCH 007/129] [3.9] gh-113659: Skip hidden .pth files (GH-113660) (GH-114146) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit (cherry picked from commit 74208ed0c440244fb809d8acc97cb9ef51e888e3) Co-authored-by: Łukasz Langa (cherry picked from commit 8fc8c45b6717be58ad927def1bf3ea05c83cab8c) 3.7.17.x: 3.7 has no io.open_code(), so .pth files are still opened with open(). --- Lib/site.py | 11 +++++- Lib/test/test_site.py | 39 +++++++++++++++++++ ...-01-02-19-52-23.gh-issue-113659.DkmnQc.rst | 1 + 3 files changed, 50 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Security/2024-01-02-19-52-23.gh-issue-113659.DkmnQc.rst diff --git a/Lib/site.py b/Lib/site.py index d1e55610ddcee48..3ed8444e75baa4c 100644 --- a/Lib/site.py +++ b/Lib/site.py @@ -73,6 +73,7 @@ import os import builtins import _sitebuiltins +import stat # Prefixes for site-packages; add additional prefixes like /usr/local here PREFIXES = [sys.prefix, sys.exec_prefix] @@ -155,6 +156,13 @@ def addpackage(sitedir, name, known_paths): else: reset = False fullname = os.path.join(sitedir, name) + try: + st = os.lstat(fullname) + except OSError: + return + if ((getattr(st, 'st_flags', 0) & stat.UF_HIDDEN) or + (getattr(st, 'st_file_attributes', 0) & stat.FILE_ATTRIBUTE_HIDDEN)): + return try: f = open(fullname, "r") except OSError: @@ -202,7 +210,8 @@ def addsitedir(sitedir, known_paths=None): names = os.listdir(sitedir) except OSError: return - names = [name for name in names if name.endswith(".pth")] + names = [name for name in names + if name.endswith(".pth") and not name.startswith(".")] for name in sorted(names): addpackage(sitedir, name, known_paths) if reset: diff --git a/Lib/test/test_site.py b/Lib/test/test_site.py index 8815c839984e918..c5af2cf9046c302 100644 --- a/Lib/test/test_site.py +++ b/Lib/test/test_site.py @@ -18,6 +18,7 @@ import urllib.request import urllib.error import shutil +import stat import subprocess import sysconfig import tempfile @@ -184,6 +185,44 @@ def test_addsitedir(self): finally: pth_file.cleanup() + def test_addsitedir_dotfile(self): + pth_file = PthFile('.dotfile') + pth_file.cleanup(prep=True) + try: + pth_file.create() + site.addsitedir(pth_file.base_dir, set()) + self.assertNotIn(site.makepath(pth_file.good_dir_path)[0], sys.path) + self.assertIn(pth_file.base_dir, sys.path) + finally: + pth_file.cleanup() + + @unittest.skipUnless(hasattr(os, 'chflags'), 'test needs os.chflags()') + def test_addsitedir_hidden_flags(self): + pth_file = PthFile() + pth_file.cleanup(prep=True) + try: + pth_file.create() + st = os.stat(pth_file.file_path) + os.chflags(pth_file.file_path, st.st_flags | stat.UF_HIDDEN) + site.addsitedir(pth_file.base_dir, set()) + self.assertNotIn(site.makepath(pth_file.good_dir_path)[0], sys.path) + self.assertIn(pth_file.base_dir, sys.path) + finally: + pth_file.cleanup() + + @unittest.skipUnless(sys.platform == 'win32', 'test needs Windows') + def test_addsitedir_hidden_file_attribute(self): + pth_file = PthFile() + pth_file.cleanup(prep=True) + try: + pth_file.create() + subprocess.check_call(['attrib', '+H', pth_file.file_path]) + site.addsitedir(pth_file.base_dir, set()) + self.assertNotIn(site.makepath(pth_file.good_dir_path)[0], sys.path) + self.assertIn(pth_file.base_dir, sys.path) + finally: + pth_file.cleanup() + # This tests _getuserbase, hence the double underline # to distinguish from a test for getuserbase def test__getuserbase(self): diff --git a/Misc/NEWS.d/next/Security/2024-01-02-19-52-23.gh-issue-113659.DkmnQc.rst b/Misc/NEWS.d/next/Security/2024-01-02-19-52-23.gh-issue-113659.DkmnQc.rst new file mode 100644 index 000000000000000..744687e72324d13 --- /dev/null +++ b/Misc/NEWS.d/next/Security/2024-01-02-19-52-23.gh-issue-113659.DkmnQc.rst @@ -0,0 +1 @@ +Skip ``.pth`` files with names starting with a dot or hidden file attribute. From 95b87885e4977076e840d3389c237b54d297004e Mon Sep 17 00:00:00 2001 From: Steve Dower Date: Fri, 24 May 2024 18:27:01 +0100 Subject: [PATCH 008/129] [3.9] gh-118486: Support mkdir(mode=0o700) on Windows (GH-118488) (GH-118741) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Łukasz Langa (cherry picked from commit 5130731c9e779b97d00a24f54cdce73ce9975dfd) 3.7.17.x: hand-merged into 3.7's os_mkdir_impl (no PySys_Audit or mkdirat probing in 3.7); the test uses slicing instead of str.removeprefix(). --- Doc/library/os.rst | 7 ++++ Lib/test/test_os.py | 12 ++++++ Lib/test/test_tempfile.py | 28 +++++++++++++ ...-05-01-20-57-09.gh-issue-118486.K44KJG.rst | 4 ++ Modules/posixmodule.c | 41 ++++++++++++++++++- 5 files changed, 90 insertions(+), 2 deletions(-) create mode 100644 Misc/NEWS.d/next/Security/2024-05-01-20-57-09.gh-issue-118486.K44KJG.rst diff --git a/Doc/library/os.rst b/Doc/library/os.rst index 685d5608f7fbfcc..f85d181ff3d0219 100644 --- a/Doc/library/os.rst +++ b/Doc/library/os.rst @@ -1833,6 +1833,10 @@ features: platform-dependent. On some platforms, they are ignored and you should call :func:`chmod` explicitly to set them. + On Windows, a *mode* of ``0o700`` is specifically handled to apply access + control to the new directory such that only the current user and + administrators have access. Other values of *mode* are ignored. + This function can also support :ref:`paths relative to directory descriptors `. @@ -1845,6 +1849,9 @@ features: .. versionchanged:: 3.6 Accepts a :term:`path-like object`. + .. versionchanged:: 3.9.20 + Windows now handles a *mode* of ``0o700``. + .. function:: makedirs(name, mode=0o777, exist_ok=False) diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py index 3a76940a089475b..58aa9d7f18a17e1 100644 --- a/Lib/test/test_os.py +++ b/Lib/test/test_os.py @@ -1200,6 +1200,18 @@ def test_exist_ok_existing_regular_file(self): self.assertRaises(OSError, os.makedirs, path, exist_ok=True) os.remove(path) + @unittest.skipUnless(os.name == 'nt', "requires Windows") + def test_win32_mkdir_700(self): + base = support.TESTFN + path = os.path.abspath(os.path.join(support.TESTFN, 'dir')) + os.mkdir(path, mode=0o700) + out = subprocess.check_output(["cacls.exe", path, "/s"], encoding="oem") + os.rmdir(path) + self.assertEqual( + out.strip(), + f'{path} "D:P(A;OICI;FA;;;SY)(A;OICI;FA;;;BA)(A;OICI;FA;;;OW)"', + ) + def tearDown(self): path = os.path.join(support.TESTFN, 'dir1', 'dir2', 'dir3', 'dir4', 'dir5', 'dir6') diff --git a/Lib/test/test_tempfile.py b/Lib/test/test_tempfile.py index 63f1412629c366e..44d8f4d0f5e5138 100644 --- a/Lib/test/test_tempfile.py +++ b/Lib/test/test_tempfile.py @@ -11,6 +11,7 @@ import warnings import contextlib import weakref +import subprocess from unittest import mock import unittest @@ -769,6 +770,33 @@ def test_mode(self): finally: os.rmdir(dir) + @unittest.skipUnless(os.name == "nt", "Only on Windows.") + def test_mode_win32(self): + # Use icacls.exe to extract the users with some level of access + # Main thing we are testing is that the BUILTIN\Users group has + # no access. The exact ACL is going to vary based on which user + # is running the test. + dir = self.do_create() + try: + out = subprocess.check_output(["icacls.exe", dir], encoding="oem").casefold() + finally: + os.rmdir(dir) + + dir = dir.casefold() + users = set() + found_user = False + for line in out.strip().splitlines(): + acl = None + # First line of result includes our directory + if line.startswith(dir): + acl = line[len(dir):].strip() + elif line and line[:1].isspace(): + acl = line.strip() + if acl: + users.add(acl.partition(":")[0]) + + self.assertNotIn(r"BUILTIN\Users".casefold(), users) + def test_collision_with_existing_file(self): # mkdtemp tries another name when a file with # the chosen name already exists diff --git a/Misc/NEWS.d/next/Security/2024-05-01-20-57-09.gh-issue-118486.K44KJG.rst b/Misc/NEWS.d/next/Security/2024-05-01-20-57-09.gh-issue-118486.K44KJG.rst new file mode 100644 index 000000000000000..a28a4e5cdb69913 --- /dev/null +++ b/Misc/NEWS.d/next/Security/2024-05-01-20-57-09.gh-issue-118486.K44KJG.rst @@ -0,0 +1,4 @@ +:func:`os.mkdir` on Windows now accepts *mode* of ``0o700`` to restrict +the new directory to the current user. This fixes CVE-2024-4030 +affecting :func:`tempfile.mkdtemp` in scenarios where the base temporary +directory is more permissive than the default. diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 43d4302b92de756..763ebd65c89222b 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -27,6 +27,11 @@ #include "Python.h" #include "pythread.h" #include "structmember.h" + +#ifdef MS_WINDOWS +# include // SetEntriesInAcl +# include // SDDL_REVISION_1 +#endif #ifndef MS_WINDOWS #include "posixmodule.h" #else @@ -3916,14 +3921,46 @@ os_mkdir_impl(PyObject *module, path_t *path, int mode, int dir_fd) /*[clinic end generated code: output=a70446903abe821f input=e965f68377e9b1ce]*/ { int result; +#ifdef MS_WINDOWS + int error = 0; + SECURITY_ATTRIBUTES secAttr = { sizeof(secAttr) }; + SECURITY_ATTRIBUTES *pSecAttr = NULL; +#endif #ifdef MS_WINDOWS Py_BEGIN_ALLOW_THREADS - result = CreateDirectoryW(path->wide, NULL); + if (mode == 0700 /* 0o700 */) { + ULONG sdSize; + pSecAttr = &secAttr; + // Set a discretionary ACL (D) that is protected (P) and includes + // inheritable (OICI) entries that allow (A) full control (FA) to + // SYSTEM (SY), Administrators (BA), and the owner (OW). + if (!ConvertStringSecurityDescriptorToSecurityDescriptorW( + L"D:P(A;OICI;FA;;;SY)(A;OICI;FA;;;BA)(A;OICI;FA;;;OW)", + SDDL_REVISION_1, + &secAttr.lpSecurityDescriptor, + &sdSize + )) { + error = GetLastError(); + } + } + if (!error) { + result = CreateDirectoryW(path->wide, pSecAttr); + if (secAttr.lpSecurityDescriptor && + // uncommonly, LocalFree returns non-zero on error, but still uses + // GetLastError() to see what the error code is + LocalFree(secAttr.lpSecurityDescriptor)) { + error = GetLastError(); + } + } Py_END_ALLOW_THREADS - if (!result) + if (error) { + return PyErr_SetFromWindowsErr(error); + } + if (!result) { return path_error(path); + } #else Py_BEGIN_ALLOW_THREADS #if HAVE_MKDIRAT From 39dab4fc2193547e62f2b79e6f944def8fc55585 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Tue, 30 Jul 2024 14:44:26 +0200 Subject: [PATCH 009/129] [3.9] gh-122133: Authenticate socket connection for `socket.socketpair()` fallback (GH-122134) (#122428) Authenticate socket connection for `socket.socketpair()` fallback when the platform does not have a native `socketpair` C API. We authenticate in-process using `getsocketname` and `getpeername` (thanks to Nathaniel J Smith for that suggestion). (cherry picked from commit 78df1043dbdce5c989600616f9f87b4ee72944e5) Co-authored-by: Seth Michael Larson Co-authored-by: Gregory P. Smith (cherry picked from commit 06fa244666ec6335a3b9bf2367e31b42b9a89b20) 3.7.17.x: tests use test.support.IPV6_ENABLED (no test.support.socket_helper in 3.7). --- Lib/socket.py | 17 +++ Lib/test/test_socket.py | 128 +++++++++++++++++- ...-07-22-13-11-28.gh-issue-122133.0mPeta.rst | 5 + 3 files changed, 147 insertions(+), 3 deletions(-) create mode 100644 Misc/NEWS.d/next/Security/2024-07-22-13-11-28.gh-issue-122133.0mPeta.rst diff --git a/Lib/socket.py b/Lib/socket.py index 40c7636cef00645..ff3c8402e58dbc8 100644 --- a/Lib/socket.py +++ b/Lib/socket.py @@ -531,6 +531,23 @@ def socketpair(family=AF_INET, type=SOCK_STREAM, proto=0): raise finally: lsock.close() + + # Authenticating avoids using a connection from something else + # able to connect to {host}:{port} instead of us. + # We expect only AF_INET and AF_INET6 families. + try: + if ( + ssock.getsockname() != csock.getpeername() + or csock.getsockname() != ssock.getpeername() + ): + raise ConnectionError("Unexpected peer connection") + except: + # getsockname() and getpeername() can fail + # if either socket isn't connected. + ssock.close() + csock.close() + raise + return (ssock, csock) __all__.append("socketpair") diff --git a/Lib/test/test_socket.py b/Lib/test/test_socket.py index 5f4b019c048d9cd..d1fa0f0ed210287 100644 --- a/Lib/test/test_socket.py +++ b/Lib/test/test_socket.py @@ -488,19 +488,27 @@ class SocketPairTest(unittest.TestCase, ThreadableTest): def __init__(self, methodName='runTest'): unittest.TestCase.__init__(self, methodName=methodName) ThreadableTest.__init__(self) + self.cli = None + self.serv = None + + def socketpair(self): + # To be overridden by some child classes. + return socket.socketpair() def setUp(self): - self.serv, self.cli = socket.socketpair() + self.serv, self.cli = self.socketpair() def tearDown(self): - self.serv.close() + if self.serv: + self.serv.close() self.serv = None def clientSetUp(self): pass def clientTearDown(self): - self.cli.close() + if self.cli: + self.cli.close() self.cli = None ThreadableTest.clientTearDown(self) @@ -4176,6 +4184,120 @@ def _testSend(self): self.assertEqual(msg, MSG) +class PurePythonSocketPairTest(SocketPairTest): + + # Explicitly use socketpair AF_INET or AF_INET6 to ensure that is the + # code path we're using regardless platform is the pure python one where + # `_socket.socketpair` does not exist. (AF_INET does not work with + # _socket.socketpair on many platforms). + def socketpair(self): + # called by super().setUp(). + try: + return socket.socketpair(socket.AF_INET6) + except OSError: + return socket.socketpair(socket.AF_INET) + + # Local imports in this class make for easy security fix backporting. + + def setUp(self): + import _socket + self._orig_sp = getattr(_socket, 'socketpair', None) + if self._orig_sp is not None: + # This forces the version using the non-OS provided socketpair + # emulation via an AF_INET socket in Lib/socket.py. + del _socket.socketpair + import importlib + global socket + socket = importlib.reload(socket) + else: + pass # This platform already uses the non-OS provided version. + super().setUp() + + def tearDown(self): + super().tearDown() + import _socket + if self._orig_sp is not None: + # Restore the default socket.socketpair definition. + _socket.socketpair = self._orig_sp + import importlib + global socket + socket = importlib.reload(socket) + + def test_recv(self): + msg = self.serv.recv(1024) + self.assertEqual(msg, MSG) + + def _test_recv(self): + self.cli.send(MSG) + + def test_send(self): + self.serv.send(MSG) + + def _test_send(self): + msg = self.cli.recv(1024) + self.assertEqual(msg, MSG) + + def test_ipv4(self): + cli, srv = socket.socketpair(socket.AF_INET) + cli.close() + srv.close() + + def _test_ipv4(self): + pass + + @unittest.skipIf(not hasattr(_socket, 'IPPROTO_IPV6') or + not hasattr(_socket, 'IPV6_V6ONLY'), + "IPV6_V6ONLY option not supported") + @unittest.skipUnless(support.IPV6_ENABLED, 'IPv6 required for this test') + def test_ipv6(self): + cli, srv = socket.socketpair(socket.AF_INET6) + cli.close() + srv.close() + + def _test_ipv6(self): + pass + + def test_injected_authentication_failure(self): + orig_getsockname = socket.socket.getsockname + inject_sock = None + + def inject_getsocketname(self): + nonlocal inject_sock + sockname = orig_getsockname(self) + # Connect to the listening socket ahead of the + # client socket. + if inject_sock is None: + inject_sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + inject_sock.setblocking(False) + try: + inject_sock.connect(sockname[:2]) + except (BlockingIOError, InterruptedError): + pass + inject_sock.setblocking(True) + return sockname + + sock1 = sock2 = None + try: + socket.socket.getsockname = inject_getsocketname + with self.assertRaises(OSError): + sock1, sock2 = socket.socketpair() + finally: + socket.socket.getsockname = orig_getsockname + if inject_sock: + inject_sock.close() + if sock1: # This cleanup isn't needed on a successful test. + sock1.close() + if sock2: + sock2.close() + + def _test_injected_authentication_failure(self): + # No-op. Exists for base class threading infrastructure to call. + # We could refactor this test into its own lesser class along with the + # setUp and tearDown code to construct an ideal; it is simpler to keep + # it here and live with extra overhead one this _one_ failure test. + pass + + class NonBlockingTCPTests(ThreadedTCPSocketTest): def __init__(self, methodName='runTest'): diff --git a/Misc/NEWS.d/next/Security/2024-07-22-13-11-28.gh-issue-122133.0mPeta.rst b/Misc/NEWS.d/next/Security/2024-07-22-13-11-28.gh-issue-122133.0mPeta.rst new file mode 100644 index 000000000000000..3544eb3824d0dac --- /dev/null +++ b/Misc/NEWS.d/next/Security/2024-07-22-13-11-28.gh-issue-122133.0mPeta.rst @@ -0,0 +1,5 @@ +Authenticate the socket connection for the ``socket.socketpair()`` fallback +on platforms where ``AF_UNIX`` is not available like Windows. + +Patch by Gregory P. Smith and Seth Larson . Reported by Ellie + From 542ae1b8cbce67b5e6fe2441ca151b3fc2dfd150 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Fri, 2 Aug 2024 15:10:52 +0200 Subject: [PATCH 010/129] [3.9] gh-122133: Rework pure Python socketpair tests to avoid use of importlib.reload. (GH-122493) (GH-122508) (cherry picked from commit f071f01b7b7e19d7d6b3a4b0ec62f820ecb14660) Co-authored-by: Russell Keith-Magee Co-authored-by: Gregory P. Smith (cherry picked from commit 3f5d9d12c74787fbf3f5891835c85cc15526c86d) --- Lib/socket.py | 121 +++++++++++++++++++--------------------- Lib/test/test_socket.py | 20 ++----- 2 files changed, 64 insertions(+), 77 deletions(-) diff --git a/Lib/socket.py b/Lib/socket.py index ff3c8402e58dbc8..1c765a52e032a51 100644 --- a/Lib/socket.py +++ b/Lib/socket.py @@ -473,16 +473,65 @@ def fromshare(info): return socket(0, 0, 0, info) __all__.append("fromshare") -if hasattr(_socket, "socketpair"): +# Origin: https://gist.github.com/4325783, by Geert Jansen. Public domain. +# This is used if _socket doesn't natively provide socketpair. It's +# always defined so that it can be patched in for testing purposes. +def _fallback_socketpair(family=AF_INET, type=SOCK_STREAM, proto=0): + if family == AF_INET: + host = _LOCALHOST + elif family == AF_INET6: + host = _LOCALHOST_V6 + else: + raise ValueError("Only AF_INET and AF_INET6 socket address families " + "are supported") + if type != SOCK_STREAM: + raise ValueError("Only SOCK_STREAM socket type is supported") + if proto != 0: + raise ValueError("Only protocol zero is supported") + + # We create a connected TCP socket. Note the trick with + # setblocking(False) that prevents us from having to create a thread. + lsock = socket(family, type, proto) + try: + lsock.bind((host, 0)) + lsock.listen() + # On IPv6, ignore flow_info and scope_id + addr, port = lsock.getsockname()[:2] + csock = socket(family, type, proto) + try: + csock.setblocking(False) + try: + csock.connect((addr, port)) + except (BlockingIOError, InterruptedError): + pass + csock.setblocking(True) + ssock, _ = lsock.accept() + except: + csock.close() + raise + finally: + lsock.close() - def socketpair(family=None, type=SOCK_STREAM, proto=0): - """socketpair([family[, type[, proto]]]) -> (socket object, socket object) + # Authenticating avoids using a connection from something else + # able to connect to {host}:{port} instead of us. + # We expect only AF_INET and AF_INET6 families. + try: + if ( + ssock.getsockname() != csock.getpeername() + or csock.getsockname() != ssock.getpeername() + ): + raise ConnectionError("Unexpected peer connection") + except: + # getsockname() and getpeername() can fail + # if either socket isn't connected. + ssock.close() + csock.close() + raise + + return (ssock, csock) - Create a pair of socket objects from the sockets returned by the platform - socketpair() function. - The arguments are the same as for socket() except the default family is - AF_UNIX if defined on the platform; otherwise, the default is AF_INET. - """ +if hasattr(_socket, "socketpair"): + def socketpair(family=None, type=SOCK_STREAM, proto=0): if family is None: try: family = AF_UNIX @@ -494,61 +543,7 @@ def socketpair(family=None, type=SOCK_STREAM, proto=0): return a, b else: - - # Origin: https://gist.github.com/4325783, by Geert Jansen. Public domain. - def socketpair(family=AF_INET, type=SOCK_STREAM, proto=0): - if family == AF_INET: - host = _LOCALHOST - elif family == AF_INET6: - host = _LOCALHOST_V6 - else: - raise ValueError("Only AF_INET and AF_INET6 socket address families " - "are supported") - if type != SOCK_STREAM: - raise ValueError("Only SOCK_STREAM socket type is supported") - if proto != 0: - raise ValueError("Only protocol zero is supported") - - # We create a connected TCP socket. Note the trick with - # setblocking(False) that prevents us from having to create a thread. - lsock = socket(family, type, proto) - try: - lsock.bind((host, 0)) - lsock.listen() - # On IPv6, ignore flow_info and scope_id - addr, port = lsock.getsockname()[:2] - csock = socket(family, type, proto) - try: - csock.setblocking(False) - try: - csock.connect((addr, port)) - except (BlockingIOError, InterruptedError): - pass - csock.setblocking(True) - ssock, _ = lsock.accept() - except: - csock.close() - raise - finally: - lsock.close() - - # Authenticating avoids using a connection from something else - # able to connect to {host}:{port} instead of us. - # We expect only AF_INET and AF_INET6 families. - try: - if ( - ssock.getsockname() != csock.getpeername() - or csock.getsockname() != ssock.getpeername() - ): - raise ConnectionError("Unexpected peer connection") - except: - # getsockname() and getpeername() can fail - # if either socket isn't connected. - ssock.close() - csock.close() - raise - - return (ssock, csock) + socketpair = _fallback_socketpair __all__.append("socketpair") socketpair.__doc__ = """socketpair([family[, type[, proto]]]) -> (socket object, socket object) diff --git a/Lib/test/test_socket.py b/Lib/test/test_socket.py index d1fa0f0ed210287..addf0f1de029e98 100644 --- a/Lib/test/test_socket.py +++ b/Lib/test/test_socket.py @@ -4185,7 +4185,6 @@ def _testSend(self): class PurePythonSocketPairTest(SocketPairTest): - # Explicitly use socketpair AF_INET or AF_INET6 to ensure that is the # code path we're using regardless platform is the pure python one where # `_socket.socketpair` does not exist. (AF_INET does not work with @@ -4200,28 +4199,21 @@ def socketpair(self): # Local imports in this class make for easy security fix backporting. def setUp(self): - import _socket - self._orig_sp = getattr(_socket, 'socketpair', None) - if self._orig_sp is not None: + if hasattr(_socket, "socketpair"): + self._orig_sp = socket.socketpair # This forces the version using the non-OS provided socketpair # emulation via an AF_INET socket in Lib/socket.py. - del _socket.socketpair - import importlib - global socket - socket = importlib.reload(socket) + socket.socketpair = socket._fallback_socketpair else: - pass # This platform already uses the non-OS provided version. + # This platform already uses the non-OS provided version. + self._orig_sp = None super().setUp() def tearDown(self): super().tearDown() - import _socket if self._orig_sp is not None: # Restore the default socket.socketpair definition. - _socket.socketpair = self._orig_sp - import importlib - global socket - socket = importlib.reload(socket) + socket.socketpair = self._orig_sp def test_recv(self): msg = self.serv.recv(1024) From fb55ab7297d215d1afde9d95d1976240b5deb095 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Langa?= Date: Wed, 4 Sep 2024 17:39:02 +0200 Subject: [PATCH 011/129] [3.9] gh-121650: Encode newlines in headers, and verify headers are sound (GH-122233) (#122610) Per RFC 2047: > [...] these encoding schemes allow the > encoding of arbitrary octet values, mail readers that implement this > decoding should also ensure that display of the decoded data on the > recipient's terminal will not cause unwanted side-effects It seems that the "quoted-word" scheme is a valid way to include a newline character in a header value, just like we already allow undecodable bytes or control characters. They do need to be properly quoted when serialized to text, though. This should fail for custom fold() implementations that aren't careful about newlines. (cherry picked from commit 097633981879b3c9de9a1dd120d3aa585ecc2384) Co-authored-by: Petr Viktorin Co-authored-by: Bas Bloemsaat Co-authored-by: Serhiy Storchaka (cherry picked from commit f7be505d137a22528cb0fc004422c0081d5d90e6) 3.7.17.x: str.removesuffix() (3.9+) replaced by slicing in email.generator. --- Doc/library/email.errors.rst | 6 ++ Doc/library/email.policy.rst | 18 ++++++ Lib/email/_header_value_parser.py | 12 +++- Lib/email/_policybase.py | 8 +++ Lib/email/errors.py | 4 ++ Lib/email/generator.py | 13 +++- Lib/test/test_email/test_generator.py | 62 +++++++++++++++++++ Lib/test/test_email/test_policy.py | 26 ++++++++ ...-07-27-16-10-41.gh-issue-121650.nf6oc9.rst | 5 ++ 9 files changed, 150 insertions(+), 4 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-07-27-16-10-41.gh-issue-121650.nf6oc9.rst diff --git a/Doc/library/email.errors.rst b/Doc/library/email.errors.rst index f4b9f52509689ea..878c09bb0402f7f 100644 --- a/Doc/library/email.errors.rst +++ b/Doc/library/email.errors.rst @@ -59,6 +59,12 @@ The following exception classes are defined in the :mod:`email.errors` module: :class:`~email.mime.image.MIMEImage`). +.. exception:: HeaderWriteError() + + Raised when an error occurs when the :mod:`~email.generator` outputs + headers. + + Here is the list of the defects that the :class:`~email.parser.FeedParser` can find while parsing messages. Note that the defects are added to the message where the problem was found, so for example, if a message nested inside a diff --git a/Doc/library/email.policy.rst b/Doc/library/email.policy.rst index 8e7076259810f55..e4ab08d880353d6 100644 --- a/Doc/library/email.policy.rst +++ b/Doc/library/email.policy.rst @@ -229,6 +229,24 @@ added matters. To illustrate:: .. versionadded:: 3.6 + + .. attribute:: verify_generated_headers + + If ``True`` (the default), the generator will raise + :exc:`~email.errors.HeaderWriteError` instead of writing a header + that is improperly folded or delimited, such that it would + be parsed as multiple headers or joined with adjacent data. + Such headers can be generated by custom header classes or bugs + in the ``email`` module. + + As it's a security feature, this defaults to ``True`` even in the + :class:`~email.policy.Compat32` policy. + For backwards compatible, but unsafe, behavior, it must be set to + ``False`` explicitly. + + .. versionadded:: 3.9.20 + + The following :class:`Policy` method is intended to be called by code using the email library to create policy instances with custom settings: diff --git a/Lib/email/_header_value_parser.py b/Lib/email/_header_value_parser.py index b7c30c47b873e31..f83352332d1c8ce 100644 --- a/Lib/email/_header_value_parser.py +++ b/Lib/email/_header_value_parser.py @@ -93,6 +93,8 @@ ASPECIALS = TSPECIALS | set("*'%") ATTRIBUTE_ENDS = ASPECIALS | WSP EXTENDED_ATTRIBUTE_ENDS = ATTRIBUTE_ENDS - set('%') +NLSET = {'\n', '\r'} +SPECIALSNL = SPECIALS | NLSET def quote_string(value): return '"'+str(value).replace('\\', '\\\\').replace('"', r'\"')+'"' @@ -2659,9 +2661,13 @@ def _refold_parse_tree(parse_tree, *, policy): wrap_as_ew_blocked -= 1 continue tstr = str(part) - if part.token_type == 'ptext' and set(tstr) & SPECIALS: - # Encode if tstr contains special characters. - want_encoding = True + if not want_encoding: + if part.token_type == 'ptext': + # Encode if tstr contains special characters. + want_encoding = not SPECIALSNL.isdisjoint(tstr) + else: + # Encode if tstr contains newlines. + want_encoding = not NLSET.isdisjoint(tstr) try: tstr.encode(encoding) charset = encoding diff --git a/Lib/email/_policybase.py b/Lib/email/_policybase.py index c9cbadd2a80c48e..d1f48211f909703 100644 --- a/Lib/email/_policybase.py +++ b/Lib/email/_policybase.py @@ -157,6 +157,13 @@ class Policy(_PolicyBase, metaclass=abc.ABCMeta): message_factory -- the class to use to create new message objects. If the value is None, the default is Message. + verify_generated_headers + -- if true, the generator verifies that each header + they are properly folded, so that a parser won't + treat it as multiple headers, start-of-body, or + part of another header. + This is a check against custom Header & fold() + implementations. """ raise_on_defect = False @@ -165,6 +172,7 @@ class Policy(_PolicyBase, metaclass=abc.ABCMeta): max_line_length = 78 mangle_from_ = False message_factory = None + verify_generated_headers = True def handle_defect(self, obj, defect): """Based on policy, either raise defect or call register_defect. diff --git a/Lib/email/errors.py b/Lib/email/errors.py index d28a6800104babc..1a0d5c63e60d909 100644 --- a/Lib/email/errors.py +++ b/Lib/email/errors.py @@ -29,6 +29,10 @@ class CharsetError(MessageError): """An illegal charset was given.""" +class HeaderWriteError(MessageError): + """Error while writing headers.""" + + # These are parsing defects which the parser was able to work around. class MessageDefect(ValueError): """Base class for a message defect.""" diff --git a/Lib/email/generator.py b/Lib/email/generator.py index ae670c2353c858a..27383c52b9f9ceb 100644 --- a/Lib/email/generator.py +++ b/Lib/email/generator.py @@ -14,12 +14,14 @@ from copy import deepcopy from io import StringIO, BytesIO from email.utils import _has_surrogates +from email.errors import HeaderWriteError UNDERSCORE = '_' NL = '\n' # XXX: no longer used by the code below. NLCRE = re.compile(r'\r\n|\r|\n') fcre = re.compile(r'^From ', re.MULTILINE) +NEWLINE_WITHOUT_FWSP = re.compile(r'\r\n[^ \t]|\r[^ \n\t]|\n[^ \t]') @@ -219,7 +221,16 @@ def _dispatch(self, msg): def _write_headers(self, msg): for h, v in msg.raw_items(): - self.write(self.policy.fold(h, v)) + folded = self.policy.fold(h, v) + if self.policy.verify_generated_headers: + linesep = self.policy.linesep + if not folded.endswith(self.policy.linesep): + raise HeaderWriteError( + f'folded header does not end with {linesep!r}: {folded!r}') + if NEWLINE_WITHOUT_FWSP.search(folded[:len(folded) - len(linesep)]): + raise HeaderWriteError( + f'folded header contains newline: {folded!r}') + self.write(folded) # A blank line always separates headers from body self.write(self._NL) diff --git a/Lib/test/test_email/test_generator.py b/Lib/test/test_email/test_generator.py index 89e7edeb63a8926..d29400f0ed1dbba 100644 --- a/Lib/test/test_email/test_generator.py +++ b/Lib/test/test_email/test_generator.py @@ -6,6 +6,7 @@ from email.generator import Generator, BytesGenerator from email.headerregistry import Address from email import policy +import email.errors from test.test_email import TestEmailBase, parameterize @@ -216,6 +217,44 @@ def test_rfc2231_wrapping_switches_to_default_len_if_too_narrow(self): g.flatten(msg) self.assertEqual(s.getvalue(), self.typ(expected)) + def test_keep_encoded_newlines(self): + msg = self.msgmaker(self.typ(textwrap.dedent("""\ + To: nobody + Subject: Bad subject=?UTF-8?Q?=0A?=Bcc: injection@example.com + + None + """))) + expected = textwrap.dedent("""\ + To: nobody + Subject: Bad subject=?UTF-8?Q?=0A?=Bcc: injection@example.com + + None + """) + s = self.ioclass() + g = self.genclass(s, policy=self.policy.clone(max_line_length=80)) + g.flatten(msg) + self.assertEqual(s.getvalue(), self.typ(expected)) + + def test_keep_long_encoded_newlines(self): + msg = self.msgmaker(self.typ(textwrap.dedent("""\ + To: nobody + Subject: Bad subject=?UTF-8?Q?=0A?=Bcc: injection@example.com + + None + """))) + expected = textwrap.dedent("""\ + To: nobody + Subject: Bad subject + =?utf-8?q?=0A?=Bcc: + injection@example.com + + None + """) + s = self.ioclass() + g = self.genclass(s, policy=self.policy.clone(max_line_length=30)) + g.flatten(msg) + self.assertEqual(s.getvalue(), self.typ(expected)) + class TestGenerator(TestGeneratorBase, TestEmailBase): @@ -224,6 +263,29 @@ class TestGenerator(TestGeneratorBase, TestEmailBase): ioclass = io.StringIO typ = str + def test_verify_generated_headers(self): + """gh-121650: by default the generator prevents header injection""" + class LiteralHeader(str): + name = 'Header' + def fold(self, **kwargs): + return self + + for text in ( + 'Value\r\nBad Injection\r\n', + 'NoNewLine' + ): + with self.subTest(text=text): + message = message_from_string( + "Header: Value\r\n\r\nBody", + policy=self.policy, + ) + + del message['Header'] + message['Header'] = LiteralHeader(text) + + with self.assertRaises(email.errors.HeaderWriteError): + message.as_string() + class TestBytesGenerator(TestGeneratorBase, TestEmailBase): diff --git a/Lib/test/test_email/test_policy.py b/Lib/test/test_email/test_policy.py index ebc3ce6f7682501..1c08d7cd5ce1b39 100644 --- a/Lib/test/test_email/test_policy.py +++ b/Lib/test/test_email/test_policy.py @@ -27,6 +27,7 @@ class PolicyAPITests(unittest.TestCase): 'raise_on_defect': False, 'mangle_from_': True, 'message_factory': None, + 'verify_generated_headers': True, } # These default values are the ones set on email.policy.default. # If any of these defaults change, the docs must be updated. @@ -278,6 +279,31 @@ def test_short_maxlen_error(self): with self.assertRaises(email.errors.HeaderParseError): policy.fold("Subject", subject) + def test_verify_generated_headers(self): + """Turning protection off allows header injection""" + policy = email.policy.default.clone(verify_generated_headers=False) + for text in ( + 'Header: Value\r\nBad: Injection\r\n', + 'Header: NoNewLine' + ): + with self.subTest(text=text): + message = email.message_from_string( + "Header: Value\r\n\r\nBody", + policy=policy, + ) + class LiteralHeader(str): + name = 'Header' + def fold(self, **kwargs): + return self + + del message['Header'] + message['Header'] = LiteralHeader(text) + + self.assertEqual( + message.as_string(), + f"{text}\nBody", + ) + # XXX: Need subclassing tests. # For adding subclassed objects, make sure the usual rules apply (subclass # wins), but that the order still works (right overrides left). diff --git a/Misc/NEWS.d/next/Library/2024-07-27-16-10-41.gh-issue-121650.nf6oc9.rst b/Misc/NEWS.d/next/Library/2024-07-27-16-10-41.gh-issue-121650.nf6oc9.rst new file mode 100644 index 000000000000000..83dd28d4ac575b8 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-07-27-16-10-41.gh-issue-121650.nf6oc9.rst @@ -0,0 +1,5 @@ +:mod:`email` headers with embedded newlines are now quoted on output. The +:mod:`~email.generator` will now refuse to serialize (write) headers that +are unsafely folded or delimited; see +:attr:`~email.policy.Policy.verify_generated_headers`. (Contributed by Bas +Bloemsaat and Petr Viktorin in :gh:`121650`.) From f82d701a2ad15e2dc85945289aca6c80088507c0 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 2 Dec 2024 13:36:46 +0100 Subject: [PATCH 012/129] [3.9] gh-103848: Adds checks to ensure that bracketed hosts found by urlsplit are of IPv6 or IPvFuture format (#103849) (#126976) Co-authored-by: Gregory P. Smith (cherry picked from commit 29f348e232e82938ba2165843c448c2b291504c5) Co-authored-by: JohnJamesUtley <81572567+JohnJamesUtley@users.noreply.github.com> (cherry picked from commit ddca2953191c67a12b1f19d6bca41016c6ae7132) 3.7.17.x: 3.7's urlsplit() still has a separate http: fast path, which gets the same check; IPv6 scope ids ("%zone") are validated here because 3.7's ipaddress cannot parse them (support came in 3.9). --- Lib/test/test_urlparse.py | 29 +++++++++++++++++++ Lib/urllib/parse.py | 24 +++++++++++++++ ...-04-26-09-54-25.gh-issue-103848.aDSnpR.rst | 2 ++ 3 files changed, 55 insertions(+) create mode 100644 Misc/NEWS.d/next/Library/2023-04-26-09-54-25.gh-issue-103848.aDSnpR.rst diff --git a/Lib/test/test_urlparse.py b/Lib/test/test_urlparse.py index bedf37455fee00a..099e0efc2c92916 100644 --- a/Lib/test/test_urlparse.py +++ b/Lib/test/test_urlparse.py @@ -1135,6 +1135,35 @@ def test_issue14072(self): self.assertEqual(p2.scheme, 'tel') self.assertEqual(p2.path, '+31641044153') + def test_invalid_bracketed_hosts(self): + self.assertRaises(ValueError, urllib.parse.urlsplit, 'Scheme://user@[192.0.2.146]/Path?Query') + self.assertRaises(ValueError, urllib.parse.urlsplit, 'Scheme://user@[important.com:8000]/Path?Query') + self.assertRaises(ValueError, urllib.parse.urlsplit, 'Scheme://user@[v123r.IP]/Path?Query') + self.assertRaises(ValueError, urllib.parse.urlsplit, 'Scheme://user@[v12ae]/Path?Query') + self.assertRaises(ValueError, urllib.parse.urlsplit, 'Scheme://user@[v.IP]/Path?Query') + self.assertRaises(ValueError, urllib.parse.urlsplit, 'Scheme://user@[v123.]/Path?Query') + self.assertRaises(ValueError, urllib.parse.urlsplit, 'Scheme://user@[v]/Path?Query') + self.assertRaises(ValueError, urllib.parse.urlsplit, 'Scheme://user@[0439:23af::2309::fae7:1234]/Path?Query') + self.assertRaises(ValueError, urllib.parse.urlsplit, 'Scheme://user@[0439:23af:2309::fae7:1234:2342:438e:192.0.2.146]/Path?Query') + self.assertRaises(ValueError, urllib.parse.urlsplit, 'Scheme://user@]v6a.ip[/Path') + # 3.7 parses http: URLs on a separate fast path; check it too. + self.assertRaises(ValueError, urllib.parse.urlsplit, 'http://user@[192.0.2.146]/Path?Query') + self.assertRaises(ValueError, urllib.parse.urlsplit, 'http://[important.com:8000]/Path') + + def test_splitting_bracketed_hosts(self): + p1 = urllib.parse.urlsplit('scheme://user@[v6a.ip]/path?query') + self.assertEqual(p1.hostname, 'v6a.ip') + self.assertEqual(p1.username, 'user') + self.assertEqual(p1.path, '/path') + p2 = urllib.parse.urlsplit('scheme://user@[0439:23af:2309::fae7%test]/path?query') + self.assertEqual(p2.hostname, '0439:23af:2309::fae7%test') + self.assertEqual(p2.username, 'user') + self.assertEqual(p2.path, '/path') + p3 = urllib.parse.urlsplit('scheme://user@[0439:23af:2309::fae7:1234:192.0.2.146%test]/path?query') + self.assertEqual(p3.hostname, '0439:23af:2309::fae7:1234:192.0.2.146%test') + self.assertEqual(p3.username, 'user') + self.assertEqual(p3.path, '/path') + def test_telurl_params(self): p1 = urllib.parse.urlparse('tel:123-4;phone-context=+1-650-516') self.assertEqual(p1.scheme, 'tel') diff --git a/Lib/urllib/parse.py b/Lib/urllib/parse.py index 79ea635ef87db42..f563a12d5008f16 100644 --- a/Lib/urllib/parse.py +++ b/Lib/urllib/parse.py @@ -34,6 +34,7 @@ import re import sys import collections +import ipaddress __all__ = ["urlparse", "urlunparse", "urljoin", "urldefrag", "urlsplit", "urlunsplit", "urlencode", "parse_qs", @@ -425,6 +426,23 @@ def _remove_unsafe_bytes_from_url(url): url = url.replace(b, "") return url +# Valid bracketed hosts are defined in +# https://www.rfc-editor.org/rfc/rfc3986#page-49 and https://url.spec.whatwg.org/ +def _check_bracketed_host(hostname): + if hostname.startswith('v'): + if not re.match(r"\Av[a-fA-F0-9]+\..+\Z", hostname): + raise ValueError(f"IPvFuture address is invalid") + else: + # The ipaddress module only accepts IPv6 scope ids ("%zone") from + # 3.9 on, so validate the address part here and check the scope id + # the way 3.9's IPv6Address does. + address, percent, scope_id = hostname.partition('%') + if percent and (not scope_id or '%' in scope_id): + raise ValueError("Invalid IPv6 scope id in %r" % hostname) + ip = ipaddress.ip_address(address) # Throws Value Error if not IPv6 or IPv4 + if isinstance(ip, ipaddress.IPv4Address): + raise ValueError(f"An IPv4 address cannot be in brackets") + def urlsplit(url, scheme='', allow_fragments=True): """Parse a URL into 5 components: :///?# @@ -458,6 +476,9 @@ def urlsplit(url, scheme='', allow_fragments=True): if (('[' in netloc and ']' not in netloc) or (']' in netloc and '[' not in netloc)): raise ValueError("Invalid IPv6 URL") + if '[' in netloc and ']' in netloc: + bracketed_host = netloc.partition('[')[2].partition(']')[0] + _check_bracketed_host(bracketed_host) if allow_fragments and '#' in url: url, fragment = url.split('#', 1) if '?' in url: @@ -482,6 +503,9 @@ def urlsplit(url, scheme='', allow_fragments=True): if (('[' in netloc and ']' not in netloc) or (']' in netloc and '[' not in netloc)): raise ValueError("Invalid IPv6 URL") + if '[' in netloc and ']' in netloc: + bracketed_host = netloc.partition('[')[2].partition(']')[0] + _check_bracketed_host(bracketed_host) if allow_fragments and '#' in url: url, fragment = url.split('#', 1) if '?' in url: diff --git a/Misc/NEWS.d/next/Library/2023-04-26-09-54-25.gh-issue-103848.aDSnpR.rst b/Misc/NEWS.d/next/Library/2023-04-26-09-54-25.gh-issue-103848.aDSnpR.rst new file mode 100644 index 000000000000000..81e5904aa6cca22 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-04-26-09-54-25.gh-issue-103848.aDSnpR.rst @@ -0,0 +1,2 @@ +Add checks to ensure that ``[`` bracketed ``]`` hosts found by +:func:`urllib.parse.urlsplit` are of IPv6 or IPvFuture format. From ea96755c45643bec10fe068e394ffd2e7e07c44e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Langa?= Date: Tue, 3 Dec 2024 18:42:19 +0100 Subject: [PATCH 013/129] [3.9] gh-122792: Make IPv4-mapped IPv6 address properties consistent with IPv4 (GH-122793) (GH-123819) (GH-127571) Make IPv4-mapped IPv6 address properties consistent with IPv4. (cherry picked from commit 76a1c5d18312712baed4699fe7333abb050ec9b7) (cherry picked from commit b58da409aac90123c1159916908a4c49144925ee) Co-authored-by: Seth Michael Larson (cherry picked from commit a897b4a46edb737167b68d27654640881f6c69fb) --- Lib/ipaddress.py | 27 +++++++++++++---- Lib/test/test_ipaddress.py | 30 +++++++++++++++++++ ...-08-07-10-42-13.gh-issue-122792.oiTMo9.rst | 3 ++ 3 files changed, 55 insertions(+), 5 deletions(-) create mode 100644 Misc/NEWS.d/next/Security/2024-08-07-10-42-13.gh-issue-122792.oiTMo9.rst diff --git a/Lib/ipaddress.py b/Lib/ipaddress.py index 07bb46f141d6e10..8814be02bcd29d8 100644 --- a/Lib/ipaddress.py +++ b/Lib/ipaddress.py @@ -312,7 +312,7 @@ def collapse_addresses(addresses): [IPv4Network('192.0.2.0/24')] Args: - addresses: An iterator of IPv4Network or IPv6Network objects. + addresses: An iterable of IPv4Network or IPv6Network objects. Returns: An iterator of the collapsed IPv(4|6)Network objects. @@ -1858,9 +1858,6 @@ def _string_from_ip_int(cls, ip_int=None): def _explode_shorthand_ip_string(self): """Expand a shortened IPv6 address. - Args: - ip_str: A string, the IPv6 address. - Returns: A string, the expanded IPv6 address. @@ -1953,6 +1950,9 @@ def is_multicast(self): See RFC 2373 2.7 for details. """ + ipv4_mapped = self.ipv4_mapped + if ipv4_mapped is not None: + return ipv4_mapped.is_multicast return self in self._constants._multicast_network @property @@ -1964,6 +1964,9 @@ def is_reserved(self): reserved IPv6 Network ranges. """ + ipv4_mapped = self.ipv4_mapped + if ipv4_mapped is not None: + return ipv4_mapped.is_reserved return any(self in x for x in self._constants._reserved_networks) @property @@ -1974,6 +1977,9 @@ def is_link_local(self): A boolean, True if the address is reserved per RFC 4291. """ + ipv4_mapped = self.ipv4_mapped + if ipv4_mapped is not None: + return ipv4_mapped.is_link_local return self in self._constants._linklocal_network @property @@ -2030,6 +2036,9 @@ def is_global(self): ``is_global`` has value opposite to :attr:`is_private`, except for the ``100.64.0.0/10`` IPv4 range where they are both ``False``. """ + ipv4_mapped = self.ipv4_mapped + if ipv4_mapped is not None: + return ipv4_mapped.is_global return not self.is_private @property @@ -2041,6 +2050,9 @@ def is_unspecified(self): RFC 2373 2.5.2. """ + ipv4_mapped = self.ipv4_mapped + if ipv4_mapped is not None: + return ipv4_mapped.is_unspecified return self._ip == 0 @property @@ -2052,6 +2064,9 @@ def is_loopback(self): RFC 2373 2.5.3. """ + ipv4_mapped = self.ipv4_mapped + if ipv4_mapped is not None: + return ipv4_mapped.is_loopback return self._ip == 1 @property @@ -2165,7 +2180,7 @@ def is_unspecified(self): @property def is_loopback(self): - return self._ip == 1 and self.network.is_loopback + return super().is_loopback and self.network.is_loopback class IPv6Network(_BaseV6, _BaseNetwork): @@ -2277,6 +2292,8 @@ class _IPv6Constants: IPv6Network('2001:db8::/32'), # IANA says N/A, let's consider it not globally reachable to be safe IPv6Network('2002::/16'), + # RFC 9637: https://www.rfc-editor.org/rfc/rfc9637.html#section-6-2.2 + IPv6Network('3fff::/20'), IPv6Network('fc00::/7'), IPv6Network('fe80::/10'), ] diff --git a/Lib/test/test_ipaddress.py b/Lib/test/test_ipaddress.py index 19c2e1ca26d30b5..c2e59088aed8f69 100644 --- a/Lib/test/test_ipaddress.py +++ b/Lib/test/test_ipaddress.py @@ -1924,6 +1924,36 @@ def testIpv4Mapped(self): self.assertEqual(ipaddress.ip_address('::ffff:c0a8:101').ipv4_mapped, ipaddress.ip_address('192.168.1.1')) + def testIpv4MappedProperties(self): + # Test that an IPv4 mapped IPv6 address has + # the same properties as an IPv4 address. + for addr4 in ( + "178.62.3.251", # global + "169.254.169.254", # link local + "127.0.0.1", # loopback + "224.0.0.1", # multicast + "192.168.0.1", # private + "0.0.0.0", # unspecified + "100.64.0.1", # public and not global + ): + with self.subTest(addr4): + ipv4 = ipaddress.IPv4Address(addr4) + ipv6 = ipaddress.IPv6Address(f"::ffff:{addr4}") + + self.assertEqual(ipv4.is_global, ipv6.is_global) + self.assertEqual(ipv4.is_private, ipv6.is_private) + self.assertEqual(ipv4.is_reserved, ipv6.is_reserved) + self.assertEqual(ipv4.is_multicast, ipv6.is_multicast) + self.assertEqual(ipv4.is_unspecified, ipv6.is_unspecified) + self.assertEqual(ipv4.is_link_local, ipv6.is_link_local) + self.assertEqual(ipv4.is_loopback, ipv6.is_loopback) + + def testIpv4MappedPrivateCheck(self): + self.assertEqual( + True, ipaddress.ip_address('::ffff:192.168.1.1').is_private) + self.assertEqual( + False, ipaddress.ip_address('::ffff:172.32.0.0').is_private) + def testAddrExclude(self): addr1 = ipaddress.ip_network('10.1.1.0/24') addr2 = ipaddress.ip_network('10.1.1.0/26') diff --git a/Misc/NEWS.d/next/Security/2024-08-07-10-42-13.gh-issue-122792.oiTMo9.rst b/Misc/NEWS.d/next/Security/2024-08-07-10-42-13.gh-issue-122792.oiTMo9.rst new file mode 100644 index 000000000000000..18e293ba0c03b5a --- /dev/null +++ b/Misc/NEWS.d/next/Security/2024-08-07-10-42-13.gh-issue-122792.oiTMo9.rst @@ -0,0 +1,3 @@ +Changed IPv4-mapped ``ipaddress.IPv6Address`` to consistently use the mapped IPv4 +address value for deciding properties. Properties which have their behavior fixed +are ``is_multicast``, ``is_reserved``, ``is_link_local``, ``is_global``, and ``is_unspecified``. From 5c2ea55bc61293d60abc85851cc507bc3511ceb5 Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> Date: Wed, 19 Feb 2025 15:31:01 +0200 Subject: [PATCH 014/129] [3.9] gh-119511: Fix a potential denial of service in imaplib (GH-119514) (#130248) The IMAP4 client could consume an arbitrary amount of memory when trying to connect to a malicious server, because it read a "literal" data with a single read(size) call, and BufferedReader.read() allocates the bytes object of the specified size before reading. Now the IMAP4 client reads data by chunks, therefore the amount of used memory is limited by the amount of the data actually been sent by the server. (cherry picked from commit 735f25c5e3a0f74438c86468ec4dfbe219d93c91) Co-authored-by: Serhiy Storchaka Co-authored-by: Gregory P. Smith (cherry picked from commit f116a9c15cba87f6cac5724214ce267ad27c29c0) --- Lib/imaplib.py | 11 ++++++++++- Lib/test/test_imaplib.py | 15 +++++++++++++++ ...2024-05-24-21-00-52.gh-issue-119511.jKrXQ8.rst | 7 +++++++ 3 files changed, 32 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Security/2024-05-24-21-00-52.gh-issue-119511.jKrXQ8.rst diff --git a/Lib/imaplib.py b/Lib/imaplib.py index dd237f7704aca0b..605f32577200492 100644 --- a/Lib/imaplib.py +++ b/Lib/imaplib.py @@ -52,6 +52,9 @@ # search command can be quite large, so we now use 1M. _MAXLINE = 1000000 +# Data larger than this will be read in chunks, to prevent extreme +# overallocation. +_SAFE_BUF_SIZE = 1 << 20 # Commands @@ -302,7 +305,13 @@ def open(self, host = '', port = IMAP4_PORT): def read(self, size): """Read 'size' bytes from remote.""" - return self.file.read(size) + cursize = min(size, _SAFE_BUF_SIZE) + data = self.file.read(cursize) + while cursize < size and len(data) == cursize: + delta = min(cursize, size - cursize) + data += self.file.read(delta) + cursize += delta + return data def readline(self): diff --git a/Lib/test/test_imaplib.py b/Lib/test/test_imaplib.py index 300a6d7b652b28b..0f717225ec017fd 100644 --- a/Lib/test/test_imaplib.py +++ b/Lib/test/test_imaplib.py @@ -854,6 +854,21 @@ def handle(self): self.assertRaises(imaplib.IMAP4.error, self.imap_class, *server.server_address) + @reap_threads + def test_truncated_large_literal(self): + size = 0 + class BadHandler(SimpleIMAPHandler): + def handle(self): + self._send_textline('* OK {%d}' % size) + self._send_textline('IMAP4rev1') + + for exponent in range(15, 64): + size = 1 << exponent + with self.subTest(f"size=2e{size}"): + with self.reaped_server(BadHandler) as server: + with self.assertRaises(imaplib.IMAP4.abort): + self.imap_class(*server.server_address) + @reap_threads def test_simple_with_statement(self): # simplest call diff --git a/Misc/NEWS.d/next/Security/2024-05-24-21-00-52.gh-issue-119511.jKrXQ8.rst b/Misc/NEWS.d/next/Security/2024-05-24-21-00-52.gh-issue-119511.jKrXQ8.rst new file mode 100644 index 000000000000000..f7b4031120e643e --- /dev/null +++ b/Misc/NEWS.d/next/Security/2024-05-24-21-00-52.gh-issue-119511.jKrXQ8.rst @@ -0,0 +1,7 @@ +Fix a potential denial of service in the :mod:`imaplib` module. When connecting +to a malicious server, it could cause an arbitrary amount of memory to be +allocated. On many systems this is harmless as unused virtual memory is only a +mapping, but if this hit a virtual address size limit it could lead to a +:exc:`MemoryError` or other process crash. On unusual systems or builds where +all allocated memory is touched and backed by actual ram or storage it could've +consumed resources doing so until similarly crashing. From e78171cd86968eaa8bd4a0f1796f6738d458957a Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Wed, 19 Feb 2025 14:36:40 +0100 Subject: [PATCH 015/129] [3.9] gh-105704: Disallow square brackets (`[` and `]`) in domain names for parsed URLs (GH-129418) (#129530) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit (cherry picked from commit d89a5f6a6e65511a5f6e0618c4c30a7aa5aba56a) Co-authored-by: Seth Michael Larson Co-authored-by: Peter Bierma Co-authored-by: Łukasz Langa (cherry picked from commit ff4e5c25666f63544071a6b075ae8b25c98b7a32) 3.7.17.x: the http: fast path calls _check_bracketed_netloc() too. --- Lib/test/test_urlparse.py | 37 ++++++++++++++++++- Lib/urllib/parse.py | 24 ++++++++++-- ...-01-28-14-08-03.gh-issue-105704.EnhHxu.rst | 4 ++ 3 files changed, 60 insertions(+), 5 deletions(-) create mode 100644 Misc/NEWS.d/next/Security/2025-01-28-14-08-03.gh-issue-105704.EnhHxu.rst diff --git a/Lib/test/test_urlparse.py b/Lib/test/test_urlparse.py index 099e0efc2c92916..ad0c300846132ea 100644 --- a/Lib/test/test_urlparse.py +++ b/Lib/test/test_urlparse.py @@ -1149,16 +1149,51 @@ def test_invalid_bracketed_hosts(self): # 3.7 parses http: URLs on a separate fast path; check it too. self.assertRaises(ValueError, urllib.parse.urlsplit, 'http://user@[192.0.2.146]/Path?Query') self.assertRaises(ValueError, urllib.parse.urlsplit, 'http://[important.com:8000]/Path') + self.assertRaises(ValueError, urllib.parse.urlsplit, 'scheme://prefix.[v6a.ip]') + self.assertRaises(ValueError, urllib.parse.urlsplit, 'scheme://[v6a.ip].suffix') + self.assertRaises(ValueError, urllib.parse.urlsplit, 'scheme://prefix.[v6a.ip]/') + self.assertRaises(ValueError, urllib.parse.urlsplit, 'scheme://[v6a.ip].suffix/') + self.assertRaises(ValueError, urllib.parse.urlsplit, 'scheme://prefix.[v6a.ip]?') + self.assertRaises(ValueError, urllib.parse.urlsplit, 'scheme://[v6a.ip].suffix?') + self.assertRaises(ValueError, urllib.parse.urlsplit, 'scheme://prefix.[::1]') + self.assertRaises(ValueError, urllib.parse.urlsplit, 'scheme://[::1].suffix') + self.assertRaises(ValueError, urllib.parse.urlsplit, 'scheme://prefix.[::1]/') + self.assertRaises(ValueError, urllib.parse.urlsplit, 'scheme://[::1].suffix/') + self.assertRaises(ValueError, urllib.parse.urlsplit, 'scheme://prefix.[::1]?') + self.assertRaises(ValueError, urllib.parse.urlsplit, 'scheme://[::1].suffix?') + self.assertRaises(ValueError, urllib.parse.urlsplit, 'scheme://prefix.[::1]:a') + self.assertRaises(ValueError, urllib.parse.urlsplit, 'scheme://[::1].suffix:a') + self.assertRaises(ValueError, urllib.parse.urlsplit, 'scheme://prefix.[::1]:a1') + self.assertRaises(ValueError, urllib.parse.urlsplit, 'scheme://[::1].suffix:a1') + self.assertRaises(ValueError, urllib.parse.urlsplit, 'scheme://prefix.[::1]:1a') + self.assertRaises(ValueError, urllib.parse.urlsplit, 'scheme://[::1].suffix:1a') + self.assertRaises(ValueError, urllib.parse.urlsplit, 'scheme://prefix.[::1]:') + self.assertRaises(ValueError, urllib.parse.urlsplit, 'scheme://[::1].suffix:/') + self.assertRaises(ValueError, urllib.parse.urlsplit, 'scheme://prefix.[::1]:?') + self.assertRaises(ValueError, urllib.parse.urlsplit, 'scheme://user@prefix.[v6a.ip]') + self.assertRaises(ValueError, urllib.parse.urlsplit, 'scheme://user@[v6a.ip].suffix') + self.assertRaises(ValueError, urllib.parse.urlsplit, 'scheme://[v6a.ip') + self.assertRaises(ValueError, urllib.parse.urlsplit, 'scheme://v6a.ip]') + self.assertRaises(ValueError, urllib.parse.urlsplit, 'scheme://]v6a.ip[') + self.assertRaises(ValueError, urllib.parse.urlsplit, 'scheme://]v6a.ip') + self.assertRaises(ValueError, urllib.parse.urlsplit, 'scheme://v6a.ip[') + self.assertRaises(ValueError, urllib.parse.urlsplit, 'scheme://prefix.[v6a.ip') + self.assertRaises(ValueError, urllib.parse.urlsplit, 'scheme://v6a.ip].suffix') + self.assertRaises(ValueError, urllib.parse.urlsplit, 'scheme://prefix]v6a.ip[suffix') + self.assertRaises(ValueError, urllib.parse.urlsplit, 'scheme://prefix]v6a.ip') + self.assertRaises(ValueError, urllib.parse.urlsplit, 'scheme://v6a.ip[suffix') def test_splitting_bracketed_hosts(self): - p1 = urllib.parse.urlsplit('scheme://user@[v6a.ip]/path?query') + p1 = urllib.parse.urlsplit('scheme://user@[v6a.ip]:1234/path?query') self.assertEqual(p1.hostname, 'v6a.ip') self.assertEqual(p1.username, 'user') self.assertEqual(p1.path, '/path') + self.assertEqual(p1.port, 1234) p2 = urllib.parse.urlsplit('scheme://user@[0439:23af:2309::fae7%test]/path?query') self.assertEqual(p2.hostname, '0439:23af:2309::fae7%test') self.assertEqual(p2.username, 'user') self.assertEqual(p2.path, '/path') + self.assertIs(p2.port, None) p3 = urllib.parse.urlsplit('scheme://user@[0439:23af:2309::fae7:1234:192.0.2.146%test]/path?query') self.assertEqual(p3.hostname, '0439:23af:2309::fae7:1234:192.0.2.146%test') self.assertEqual(p3.username, 'user') diff --git a/Lib/urllib/parse.py b/Lib/urllib/parse.py index f563a12d5008f16..46a3dfc58e77dad 100644 --- a/Lib/urllib/parse.py +++ b/Lib/urllib/parse.py @@ -426,6 +426,24 @@ def _remove_unsafe_bytes_from_url(url): url = url.replace(b, "") return url + +def _check_bracketed_netloc(netloc): + # Note that this function must mirror the splitting + # done in NetlocResultMixins._hostinfo(). + hostname_and_port = netloc.rpartition('@')[2] + before_bracket, have_open_br, bracketed = hostname_and_port.partition('[') + if have_open_br: + # No data is allowed before a bracket. + if before_bracket: + raise ValueError("Invalid IPv6 URL") + hostname, _, port = bracketed.partition(']') + # No data is allowed after the bracket but before the port delimiter. + if port and not port.startswith(":"): + raise ValueError("Invalid IPv6 URL") + else: + hostname, _, port = hostname_and_port.partition(':') + _check_bracketed_host(hostname) + # Valid bracketed hosts are defined in # https://www.rfc-editor.org/rfc/rfc3986#page-49 and https://url.spec.whatwg.org/ def _check_bracketed_host(hostname): @@ -477,8 +495,7 @@ def urlsplit(url, scheme='', allow_fragments=True): (']' in netloc and '[' not in netloc)): raise ValueError("Invalid IPv6 URL") if '[' in netloc and ']' in netloc: - bracketed_host = netloc.partition('[')[2].partition(']')[0] - _check_bracketed_host(bracketed_host) + _check_bracketed_netloc(netloc) if allow_fragments and '#' in url: url, fragment = url.split('#', 1) if '?' in url: @@ -504,8 +521,7 @@ def urlsplit(url, scheme='', allow_fragments=True): (']' in netloc and '[' not in netloc)): raise ValueError("Invalid IPv6 URL") if '[' in netloc and ']' in netloc: - bracketed_host = netloc.partition('[')[2].partition(']')[0] - _check_bracketed_host(bracketed_host) + _check_bracketed_netloc(netloc) if allow_fragments and '#' in url: url, fragment = url.split('#', 1) if '?' in url: diff --git a/Misc/NEWS.d/next/Security/2025-01-28-14-08-03.gh-issue-105704.EnhHxu.rst b/Misc/NEWS.d/next/Security/2025-01-28-14-08-03.gh-issue-105704.EnhHxu.rst new file mode 100644 index 000000000000000..bff1bc6b0d609c4 --- /dev/null +++ b/Misc/NEWS.d/next/Security/2025-01-28-14-08-03.gh-issue-105704.EnhHxu.rst @@ -0,0 +1,4 @@ +When using :func:`urllib.parse.urlsplit` and :func:`urllib.parse.urlparse` host +parsing would not reject domain names containing square brackets (``[`` and +``]``). Square brackets are only valid for IPv6 and IPvFuture hosts according to +`RFC 3986 Section 3.2.2 `__. From 525a263f84d518b7a13c701d3f6c6a14d09f9668 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Thu, 3 Apr 2025 18:27:33 +0200 Subject: [PATCH 016/129] [3.9] gh-121284: Fix email address header folding with parsed encoded-word (GH-122754) (GH-131412) Email generators using email.policy.default may convert an RFC 2047 encoded-word to unencoded form during header refolding. In a structured header, this could allow 'specials' chars outside a quoted-string, leading to invalid address headers and enabling spoofing. This change ensures a parsed encoded-word that contains specials is kept as an encoded-word while the header is refolded. [Better fix from @bitdancer.] (cherry picked from commit 295b53df2aa18deb625a7da41f7e4babfe6ef34b) Co-authored-by: Mike Edmunds Co-authored-by: R David Murray (cherry picked from commit 9a31386eec8715ab5842bbe8cfcc0fb5cf3631af) --- Lib/email/_header_value_parser.py | 10 ++++---- .../test_email/test__header_value_parser.py | 25 +++++++++++++++++++ ...-08-06-12-27-34.gh-issue-121284.8rwPxe.rst | 7 ++++++ 3 files changed, 37 insertions(+), 5 deletions(-) create mode 100644 Misc/NEWS.d/next/Security/2024-08-06-12-27-34.gh-issue-121284.8rwPxe.rst diff --git a/Lib/email/_header_value_parser.py b/Lib/email/_header_value_parser.py index f83352332d1c8ce..09792f555346c9b 100644 --- a/Lib/email/_header_value_parser.py +++ b/Lib/email/_header_value_parser.py @@ -1030,7 +1030,7 @@ def get_fws(value): fws = WhiteSpaceTerminal(value[:len(value)-len(newvalue)], 'fws') return fws, newvalue -def get_encoded_word(value): +def get_encoded_word(value, terminal_type='vtext'): """ encoded-word = "=?" charset "?" encoding "?" encoded-text "?=" """ @@ -1069,7 +1069,7 @@ def get_encoded_word(value): ew.append(token) continue chars, *remainder = _wsp_splitter(text, 1) - vtext = ValueTerminal(chars, 'vtext') + vtext = ValueTerminal(chars, terminal_type) _validate_xtext(vtext) ew.append(vtext) text = ''.join(remainder) @@ -1111,7 +1111,7 @@ def get_unstructured(value): valid_ew = True if value.startswith('=?'): try: - token, value = get_encoded_word(value) + token, value = get_encoded_word(value, 'utext') except _InvalidEwError: valid_ew = False except errors.HeaderParseError: @@ -1140,7 +1140,7 @@ def get_unstructured(value): # the parser to go in an infinite loop. if valid_ew and rfc2047_matcher.search(tok): tok, *remainder = value.partition('=?') - vtext = ValueTerminal(tok, 'vtext') + vtext = ValueTerminal(tok, 'utext') _validate_xtext(vtext) unstructured.append(vtext) value = ''.join(remainder) @@ -2662,7 +2662,7 @@ def _refold_parse_tree(parse_tree, *, policy): continue tstr = str(part) if not want_encoding: - if part.token_type == 'ptext': + if part.token_type in ('ptext', 'vtext'): # Encode if tstr contains special characters. want_encoding = not SPECIALSNL.isdisjoint(tstr) else: diff --git a/Lib/test/test_email/test__header_value_parser.py b/Lib/test/test_email/test__header_value_parser.py index 4894d8f22e1242a..86fd3b5ffefcb90 100644 --- a/Lib/test/test_email/test__header_value_parser.py +++ b/Lib/test/test_email/test__header_value_parser.py @@ -2828,6 +2828,31 @@ def test_address_list_with_unicode_names_in_quotes(self): '=?utf-8?q?H=C3=BCbsch?= Kaktus ,\n' ' =?utf-8?q?bei=C3=9Ft_bei=C3=9Ft?= \n') + def test_address_list_with_specials_in_encoded_word(self): + # An encoded-word parsed from a structured header must remain + # encoded when it contains specials. Regression for gh-121284. + policy = self.policy.clone(max_line_length=40) + cases = [ + # (to, folded) + ('=?utf-8?q?A_v=C3=A9ry_long_name_with=2C_comma?= ', + 'A =?utf-8?q?v=C3=A9ry_long_name_with?=\n' + ' =?utf-8?q?=2C?= comma \n'), + ('=?utf-8?q?This_long_name_does_not_need_encoded=2Dword?= ', + 'This long name does not need\n' + ' encoded-word \n'), + ('"A véry long name with, comma" ', + # (This isn't the best fold point, but it's not invalid.) + 'A =?utf-8?q?v=C3=A9ry_long_name_with?=\n' + ' =?utf-8?q?=2C?= comma \n'), + ('"A véry long name containing a, comma" ', + 'A =?utf-8?q?v=C3=A9ry?= long name\n' + ' containing =?utf-8?q?a=2C?= comma\n' + ' \n'), + ] + for (to, folded) in cases: + with self.subTest(to=to): + self._test(parser.get_address_list(to)[0], folded, policy=policy) + # XXX Need tests with comments on various sides of a unicode token, # and with unicode tokens in the comments. Spaces inside the quotes # currently don't do the right thing. diff --git a/Misc/NEWS.d/next/Security/2024-08-06-12-27-34.gh-issue-121284.8rwPxe.rst b/Misc/NEWS.d/next/Security/2024-08-06-12-27-34.gh-issue-121284.8rwPxe.rst new file mode 100644 index 000000000000000..923e91170d355f5 --- /dev/null +++ b/Misc/NEWS.d/next/Security/2024-08-06-12-27-34.gh-issue-121284.8rwPxe.rst @@ -0,0 +1,7 @@ +Fix bug in the folding of rfc2047 encoded-words when flattening an email message +using a modern email policy. Previously when an encoded-word was too long +for a line, it would be decoded, split across lines, and re-encoded. But commas +and other special characters in the original text could be left unencoded and +unquoted. This could theoretically be used to spoof header lines using +a carefully constructed encoded-word if the resulting rendered email was +transmitted or re-parsed. From 856d6cfc06254cdcf4b35b93b7658d4f7b51fc66 Mon Sep 17 00:00:00 2001 From: Brian Schubert Date: Mon, 2 Jun 2025 11:57:06 -0400 Subject: [PATCH 017/129] [3.9] gh-80222: Fix email address header folding with long quoted-string (GH-122753) (GH-129111) (GH-132371) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Email generators using email.policy.default could incorrectly omit the quote ('"') characters from a quoted-string during header refolding, leading to invalid address headers and enabling header spoofing. This change restores the quote characters on a bare-quoted-string as the header is refolded, and escapes backslash and quote chars in the string. (cherry picked from commit 5aaf4168583) (cherry picked from commit a4ef689ce670684ec132204b1cd03720c8e0a03d) Co-authored-by: R. David Murray Co-authored-by: Mike Edmunds Co-authored-by: Łukasz Langa (cherry picked from commit d4df3c55e4c5513947f907f24766b34d2ae8c090) --- Lib/email/_header_value_parser.py | 19 ++++++++++++- .../test_email/test__header_value_parser.py | 27 +++++++++++++++++++ ...4-08-06-11-43-08.gh-issue-80222.wfR4BU.rst | 6 +++++ 3 files changed, 51 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Security/2024-08-06-11-43-08.gh-issue-80222.wfR4BU.rst diff --git a/Lib/email/_header_value_parser.py b/Lib/email/_header_value_parser.py index 09792f555346c9b..d73360ed6fa1e08 100644 --- a/Lib/email/_header_value_parser.py +++ b/Lib/email/_header_value_parser.py @@ -96,8 +96,16 @@ NLSET = {'\n', '\r'} SPECIALSNL = SPECIALS | NLSET + +def make_quoted_pairs(value): + """Escape dquote and backslash for use within a quoted-string.""" + return str(value).replace('\\', '\\\\').replace('"', '\\"') + + def quote_string(value): - return '"'+str(value).replace('\\', '\\\\').replace('"', r'\"')+'"' + escaped = make_quoted_pairs(value) + return f'"{escaped}"' + # Match a RFC 2047 word, looks like =?utf-8?q?someword?= rfc2047_matcher = re.compile(r''' @@ -2729,6 +2737,15 @@ def _refold_parse_tree(parse_tree, *, policy): if not hasattr(part, 'encode'): # It's not a terminal, try folding the subparts. newparts = list(part) + if part.token_type == 'bare-quoted-string': + # To fold a quoted string we need to create a list of terminal + # tokens that will render the leading and trailing quotes + # and use quoted pairs in the value as appropriate. + newparts = ( + [ValueTerminal('"', 'ptext')] + + [ValueTerminal(make_quoted_pairs(p), 'ptext') + for p in newparts] + + [ValueTerminal('"', 'ptext')]) if not part.as_ew_allowed: wrap_as_ew_blocked += 1 newparts.append(end_ew_not_allowed) diff --git a/Lib/test/test_email/test__header_value_parser.py b/Lib/test/test_email/test__header_value_parser.py index 86fd3b5ffefcb90..9a4e8c7c5a43f5a 100644 --- a/Lib/test/test_email/test__header_value_parser.py +++ b/Lib/test/test_email/test__header_value_parser.py @@ -2828,6 +2828,33 @@ def test_address_list_with_unicode_names_in_quotes(self): '=?utf-8?q?H=C3=BCbsch?= Kaktus ,\n' ' =?utf-8?q?bei=C3=9Ft_bei=C3=9Ft?= \n') + def test_address_list_with_specials_in_long_quoted_string(self): + # Regression for gh-80222. + policy = self.policy.clone(max_line_length=40) + cases = [ + # (to, folded) + ('"Exfiltrator (unclosed comment?" ', + '"Exfiltrator (unclosed\n' + ' comment?" \n'), + ('"Escaped \\" chars \\\\ in quoted-string stay escaped" ', + '"Escaped \\" chars \\\\ in quoted-string\n' + ' stay escaped" \n'), + ('This long display name does not need quotes ', + 'This long display name does not need\n' + ' quotes \n'), + ('"Quotes are not required but are retained here" ', + '"Quotes are not required but are\n' + ' retained here" \n'), + ('"A quoted-string, it can be a valid local-part"@example.com', + '"A quoted-string, it can be a valid\n' + ' local-part"@example.com\n'), + ('"local-part-with-specials@but-no-fws.cannot-fold"@example.com', + '"local-part-with-specials@but-no-fws.cannot-fold"@example.com\n'), + ] + for (to, folded) in cases: + with self.subTest(to=to): + self._test(parser.get_address_list(to)[0], folded, policy=policy) + def test_address_list_with_specials_in_encoded_word(self): # An encoded-word parsed from a structured header must remain # encoded when it contains specials. Regression for gh-121284. diff --git a/Misc/NEWS.d/next/Security/2024-08-06-11-43-08.gh-issue-80222.wfR4BU.rst b/Misc/NEWS.d/next/Security/2024-08-06-11-43-08.gh-issue-80222.wfR4BU.rst new file mode 100644 index 000000000000000..0f0661d0b1cf4ac --- /dev/null +++ b/Misc/NEWS.d/next/Security/2024-08-06-11-43-08.gh-issue-80222.wfR4BU.rst @@ -0,0 +1,6 @@ +Fix bug in the folding of quoted strings when flattening an email message using +a modern email policy. Previously when a quoted string was folded so that +it spanned more than one line, the surrounding quotes and internal escapes +would be omitted. This could theoretically be used to spoof header lines +using a carefully constructed quoted string if the resulting rendered email +was transmitted or re-parsed. From a10ddaa180b8c6939da7a8ffbe85c5dec6a8cb71 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Thu, 29 Oct 2026 09:39:15 -0500 Subject: [PATCH 018/129] [3.7] gh-133767: Fix use-after-free in the unicode-escape decoder with an error handler (GH-129648) (GH-133944) (#134346) * [3.9] gh-133767: Fix use-after-free in the unicode-escape decoder with an error handler (GH-129648) (GH-133944) If the error handler is used, a new bytes object is created to set as the object attribute of UnicodeDecodeError, and that bytes object then replaces the original data. A pointer to the decoded data will became invalid after destroying that temporary bytes object. So we need other way to return the first invalid escape from _PyUnicode_DecodeUnicodeEscapeInternal(). _PyBytes_DecodeEscape() does not have such issue, because it does not use the error handlers registry, but it should be changed for compatibility with _PyUnicode_DecodeUnicodeEscapeInternal(). (cherry picked from commit 9f69a58623bd01349a18ba0c7a9cb1dad6a51e8e) (cherry picked from commit 6279eb8c076d89d3739a6edb393e43c7929b429d) (cherry picked from commit a75953b347716fff694aa59a7c7c2489fa50d1f5) (cherry picked from commit 0c33e5baedf18ebcb04bc41dff7cfc614d5ea5fe) (cherry picked from commit 8b528cacbbde60504f6ac62784d04889d285f18b) Co-authored-by: Serhiy Storchaka 3.7.17.x: hand-ported. 3.7 names the helper _PyUnicode_DecodeUnicodeEscape (no consumed argument); the char-returning variant is _PyUnicode_DecodeUnicodeEscape2 and the old symbol stays as a wrapper. Python/ast.c is unchanged: it passes errors=NULL, so no error handler can replace the input buffer. The _PyBytes_DecodeEscape change is not ported; upstream made it only for API symmetry (the bytes decoder never calls the error handler registry). (cherry picked from commit 8d35fd1b34935221aff23a1ab69a429dd156be77) --- Include/unicodeobject.h | 11 +++++ Lib/test/test_codeccallbacks.py | 34 ++++++++++++++ ...-05-09-20-22-54.gh-issue-133767.kN2i3Q.rst | 2 + Objects/unicodeobject.c | 44 ++++++++++++++----- 4 files changed, 80 insertions(+), 11 deletions(-) create mode 100644 Misc/NEWS.d/next/Security/2025-05-09-20-22-54.gh-issue-133767.kN2i3Q.rst diff --git a/Include/unicodeobject.h b/Include/unicodeobject.h index 45998a13a621660..2caba925af89b47 100644 --- a/Include/unicodeobject.h +++ b/Include/unicodeobject.h @@ -1501,6 +1501,17 @@ PyAPI_FUNC(PyObject*) PyUnicode_DecodeUnicodeEscape( #ifndef Py_LIMITED_API /* Helper for PyUnicode_DecodeUnicodeEscape that detects invalid escape chars. */ +PyAPI_FUNC(PyObject*) _PyUnicode_DecodeUnicodeEscape2( + const char *string, /* Unicode-Escape encoded string */ + Py_ssize_t length, /* size of string */ + const char *errors, /* error handling */ + int *first_invalid_escape_char, /* on return, if not -1, contain the first + invalid escaped char in string. */ + const char **first_invalid_escape_ptr); /* on return, if not NULL, may + point to the first invalid escaped + char in string. + May be NULL if errors is not NULL. */ +// Export for binary compatibility. PyAPI_FUNC(PyObject*) _PyUnicode_DecodeUnicodeEscape( const char *string, /* Unicode-Escape encoded string */ Py_ssize_t length, /* size of string */ diff --git a/Lib/test/test_codeccallbacks.py b/Lib/test/test_codeccallbacks.py index e2e7463389ef72c..0d91afd56c42399 100644 --- a/Lib/test/test_codeccallbacks.py +++ b/Lib/test/test_codeccallbacks.py @@ -1044,6 +1044,40 @@ def mutating(exc): for (encoding, data) in baddata: self.assertEqual(data.decode(encoding, "test.mutating"), "\u4242") + def test_mutating_decode_handler_unicode_escape(self): + decode = codecs.unicode_escape_decode + def mutating(exc): + if isinstance(exc, UnicodeDecodeError): + r = data.get(exc.object[:exc.end]) + if r is not None: + exc.object = r[0] + exc.object[exc.end:] + return ('\u0404', r[1]) + raise AssertionError("don't know how to handle %r" % exc) + + codecs.register_error('test.mutating2', mutating) + data = { + br'\x0': (b'\\', 0), + br'\x3': (b'xxx\\', 3), + br'\x5': (b'x\\', 1), + } + def check(input, expected, msg): + with self.assertWarns(DeprecationWarning) as cm: + self.assertEqual(decode(input, 'test.mutating2'), (expected, len(input))) + self.assertIn(msg, str(cm.warning)) + + check(br'\x0n\z', '\u0404\n\\z', r"invalid escape sequence '\z'") + check(br'\x0z', '\u0404\\z', r"invalid escape sequence '\z'") + + check(br'\x3n\zr', '\u0404\n\\zr', r"invalid escape sequence '\z'") + check(br'\x3zr', '\u0404\\zr', r"invalid escape sequence '\z'") + check(br'\x3z5', '\u0404\\z5', r"invalid escape sequence '\z'") + check(memoryview(br'\x3z5x')[:-1], '\u0404\\z5', r"invalid escape sequence '\z'") + check(memoryview(br'\x3z5xy')[:-2], '\u0404\\z5', r"invalid escape sequence '\z'") + + check(br'\x5n\z', '\u0404\n\\z', r"invalid escape sequence '\z'") + check(br'\x5z', '\u0404\\z', r"invalid escape sequence '\z'") + check(memoryview(br'\x5zy')[:-1], '\u0404\\z', r"invalid escape sequence '\z'") + # issue32583 def test_crashing_decode_handler(self): # better generating one more character to fill the extra space slot diff --git a/Misc/NEWS.d/next/Security/2025-05-09-20-22-54.gh-issue-133767.kN2i3Q.rst b/Misc/NEWS.d/next/Security/2025-05-09-20-22-54.gh-issue-133767.kN2i3Q.rst new file mode 100644 index 000000000000000..39d2f1e1a892cf3 --- /dev/null +++ b/Misc/NEWS.d/next/Security/2025-05-09-20-22-54.gh-issue-133767.kN2i3Q.rst @@ -0,0 +1,2 @@ +Fix use-after-free in the "unicode-escape" decoder with a non-"strict" error +handler. diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index d7e5a9d9cc98f2e..9c6d34eab09797a 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -5918,19 +5918,22 @@ PyUnicode_AsUTF16String(PyObject *unicode) static _PyUnicode_Name_CAPI *ucnhash_CAPI = NULL; PyObject * -_PyUnicode_DecodeUnicodeEscape(const char *s, +_PyUnicode_DecodeUnicodeEscape2(const char *s, Py_ssize_t size, const char *errors, - const char **first_invalid_escape) + int *first_invalid_escape_char, + const char **first_invalid_escape_ptr) { const char *starts = s; + const char *initial_starts = starts; _PyUnicodeWriter writer; const char *end; PyObject *errorHandler = NULL; PyObject *exc = NULL; // so we can remember if we've seen an invalid escape char or not - *first_invalid_escape = NULL; + *first_invalid_escape_char = -1; + *first_invalid_escape_ptr = NULL; if (size == 0) { _Py_RETURN_UNICODE_EMPTY(); @@ -6106,9 +6109,12 @@ _PyUnicode_DecodeUnicodeEscape(const char *s, goto error; default: - if (*first_invalid_escape == NULL) { - *first_invalid_escape = s-1; /* Back up one char, since we've - already incremented s. */ + if (*first_invalid_escape_char == -1) { + *first_invalid_escape_char = c; + if (starts == initial_starts) { + /* Back up one char, since we've already incremented s. */ + *first_invalid_escape_ptr = s - 1; + } } WRITE_ASCII_CHAR('\\'); WRITE_CHAR(c); @@ -6142,20 +6148,36 @@ _PyUnicode_DecodeUnicodeEscape(const char *s, return NULL; } +// Export for binary compatibility. +PyObject * +_PyUnicode_DecodeUnicodeEscape(const char *s, + Py_ssize_t size, + const char *errors, + const char **first_invalid_escape) +{ + int first_invalid_escape_char; + return _PyUnicode_DecodeUnicodeEscape2( + s, size, errors, + &first_invalid_escape_char, + first_invalid_escape); +} + PyObject * PyUnicode_DecodeUnicodeEscape(const char *s, Py_ssize_t size, const char *errors) { - const char *first_invalid_escape; - PyObject *result = _PyUnicode_DecodeUnicodeEscape(s, size, errors, - &first_invalid_escape); + int first_invalid_escape_char; + const char *first_invalid_escape_ptr; + PyObject *result = _PyUnicode_DecodeUnicodeEscape2(s, size, errors, + &first_invalid_escape_char, + &first_invalid_escape_ptr); if (result == NULL) return NULL; - if (first_invalid_escape != NULL) { + if (first_invalid_escape_char != -1) { if (PyErr_WarnFormat(PyExc_DeprecationWarning, 1, "invalid escape sequence '\\%c'", - (unsigned char)*first_invalid_escape) < 0) { + first_invalid_escape_char) < 0) { Py_DECREF(result); return NULL; } From f36450d99182abe3926c0a01336ec212cb9013c6 Mon Sep 17 00:00:00 2001 From: icanhasmath Date: Thu, 29 Oct 2026 09:48:06 -0500 Subject: [PATCH 019/129] 3.7.17.x: update testPrivateNetworks for the CVE-2024-4032 fix The CVE-2024-4032 backport (89f48de7b77) added 0.0.0.0/8 and 255.255.255.255/32 to the private ranges but kept the old assertion that 0.0.0.0/0 is not private. Upstream changed that assertion to True in the same fix (GH-113179); 3.9 and later expect True too. Test-only. --- Lib/test/test_ipaddress.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_ipaddress.py b/Lib/test/test_ipaddress.py index c2e59088aed8f69..a047d9b4aa5f875 100644 --- a/Lib/test/test_ipaddress.py +++ b/Lib/test/test_ipaddress.py @@ -1792,7 +1792,7 @@ def testReservedIpv4(self): self.assertEqual(True, ipaddress.ip_network('0.0.0.0').is_unspecified) def testPrivateNetworks(self): - self.assertEqual(False, ipaddress.ip_network("0.0.0.0/0").is_private) + self.assertEqual(True, ipaddress.ip_network("0.0.0.0/0").is_private) self.assertEqual(False, ipaddress.ip_network("1.0.0.0/8").is_private) self.assertEqual(True, ipaddress.ip_network("0.0.0.0/8").is_private) From 2354baeb07bcaa9ace987ba1624073e1d7e3971d Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Tue, 3 Jun 2025 15:55:59 +0200 Subject: [PATCH 020/129] [3.9] gh-128840: Limit the number of parts in IPv6 address parsing (GH-128841) (GH-134615) Limit length of IP address string to 39 (cherry picked from commit 47f1161d3a2bec52b5b5e952150141709c247da2) Co-authored-by: Seth Michael Larson Co-authored-by: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> Co-authored-by: Serhiy Storchaka Co-authored-by: Gregory P. Smith (cherry picked from commit 78394a612b3c3bf4b84fdfd484623bfbcdeef02b) 3.7.17.x: scope-id test cases dropped (3.7's ipaddress has no scope ids). --- Lib/ipaddress.py | 11 +++++++++-- Lib/test/test_ipaddress.py | 10 ++++++++++ .../2025-01-14-11-19-07.gh-issue-128840.M1doZW.rst | 2 ++ 3 files changed, 21 insertions(+), 2 deletions(-) create mode 100644 Misc/NEWS.d/next/Security/2025-01-14-11-19-07.gh-issue-128840.M1doZW.rst diff --git a/Lib/ipaddress.py b/Lib/ipaddress.py index 8814be02bcd29d8..99108fdd1ea2bc3 100644 --- a/Lib/ipaddress.py +++ b/Lib/ipaddress.py @@ -1667,8 +1667,16 @@ def _ip_int_from_string(cls, ip_str): """ if not ip_str: raise AddressValueError('Address cannot be empty') + if len(ip_str) > 39: + msg = ("At most 39 characters expected in " + f"{ip_str[:14]!r}({len(ip_str)-28} chars elided){ip_str[-14:]!r}") + raise AddressValueError(msg) - parts = ip_str.split(':') + # We want to allow more parts than the max to be 'split' + # to preserve the correct error message when there are + # too many parts combined with '::' + _max_parts = cls._HEXTET_COUNT + 1 + parts = ip_str.split(':', maxsplit=_max_parts) # An IPv6 address needs at least 2 colons (3 parts). _min_parts = 3 @@ -1688,7 +1696,6 @@ def _ip_int_from_string(cls, ip_str): # An IPv6 address can't have more than 8 colons (9 parts). # The extra colon comes from using the "::" notation for a single # leading or trailing zero part. - _max_parts = cls._HEXTET_COUNT + 1 if len(parts) > _max_parts: msg = "At most %d colons permitted in %r" % (_max_parts-1, ip_str) raise AddressValueError(msg) diff --git a/Lib/test/test_ipaddress.py b/Lib/test/test_ipaddress.py index a047d9b4aa5f875..71aac76d6a63a49 100644 --- a/Lib/test/test_ipaddress.py +++ b/Lib/test/test_ipaddress.py @@ -301,6 +301,16 @@ def assertBadSplit(addr): # A trailing IPv4 address is two parts assertBadSplit("10:9:8:7:6:5:4:3:42.42.42.42") + def test_bad_address_split_v6_too_long(self): + def assertBadSplit(addr): + msg = r"At most 39 characters expected in %s" + with self.assertAddressError(msg, repr(re.escape(addr[:14]))): + ipaddress.IPv6Address(addr) + + # Long IPv6 address + long_addr = ("0:" * 10000) + "0" + assertBadSplit(long_addr) + def test_bad_address_split_v6_too_many_parts(self): def assertBadSplit(addr): msg = "Exactly 8 parts expected without '::' in %r" diff --git a/Misc/NEWS.d/next/Security/2025-01-14-11-19-07.gh-issue-128840.M1doZW.rst b/Misc/NEWS.d/next/Security/2025-01-14-11-19-07.gh-issue-128840.M1doZW.rst new file mode 100644 index 000000000000000..b57ec3e70dcc5fa --- /dev/null +++ b/Misc/NEWS.d/next/Security/2025-01-14-11-19-07.gh-issue-128840.M1doZW.rst @@ -0,0 +1,2 @@ +Short-circuit the processing of long IPv6 addresses early in :mod:`ipaddress` to prevent excessive +memory consumption and a minor denial-of-service. From fe7e9e6ec5ca67c46cd4528a9e359d1fb7ef48be Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Tue, 3 Jun 2025 16:27:03 +0200 Subject: [PATCH 021/129] [3.9] gh-128840: Fix parsing long IPv6 addresses with embedded IPv4 address (GH-134836) (GH-135090) (cherry picked from commit d83576bf48d07d5e29d5d171c4e25afb048622aa) Co-authored-by: Serhiy Storchaka (cherry picked from commit 24eaf53bc6e39fde6ad36b036768dc7d5c144d47) 3.7.17.x: scope-id test cases dropped (3.7's ipaddress has no scope ids). 3.7.17.x: the ::ffff:255.255.255.255 case expects 3.7's hex form ::ffff:ffff:ffff, because gh-87799 (dotted IPv4-mapped output) is not backported. --- Lib/ipaddress.py | 10 ++++++---- Lib/test/test_ipaddress.py | 16 ++++++++++++++-- ...025-05-28-15-53-27.gh-issue-128840.Nur2pB.rst | 1 + 3 files changed, 21 insertions(+), 6 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2025-05-28-15-53-27.gh-issue-128840.Nur2pB.rst diff --git a/Lib/ipaddress.py b/Lib/ipaddress.py index 99108fdd1ea2bc3..8fb43513ed1a22a 100644 --- a/Lib/ipaddress.py +++ b/Lib/ipaddress.py @@ -1667,10 +1667,12 @@ def _ip_int_from_string(cls, ip_str): """ if not ip_str: raise AddressValueError('Address cannot be empty') - if len(ip_str) > 39: - msg = ("At most 39 characters expected in " - f"{ip_str[:14]!r}({len(ip_str)-28} chars elided){ip_str[-14:]!r}") - raise AddressValueError(msg) + if len(ip_str) > 45: + shorten = ip_str + if len(shorten) > 100: + shorten = f'{ip_str[:45]}({len(ip_str)-90} chars elided){ip_str[-45:]}' + raise AddressValueError(f"At most 45 characters expected in " + f"{shorten!r}") # We want to allow more parts than the max to be 'split' # to preserve the correct error message when there are diff --git a/Lib/test/test_ipaddress.py b/Lib/test/test_ipaddress.py index 71aac76d6a63a49..e4ab71a25cc8093 100644 --- a/Lib/test/test_ipaddress.py +++ b/Lib/test/test_ipaddress.py @@ -303,13 +303,15 @@ def assertBadSplit(addr): def test_bad_address_split_v6_too_long(self): def assertBadSplit(addr): - msg = r"At most 39 characters expected in %s" - with self.assertAddressError(msg, repr(re.escape(addr[:14]))): + msg = r"At most 45 characters expected in '%s" + with self.assertAddressError(msg, re.escape(addr[:45])): ipaddress.IPv6Address(addr) # Long IPv6 address long_addr = ("0:" * 10000) + "0" assertBadSplit(long_addr) + assertBadSplit(long_addr + ":255.255.255.255") + assertBadSplit(long_addr + ":ffff:255.255.255.255") def test_bad_address_split_v6_too_many_parts(self): def assertBadSplit(addr): @@ -1692,6 +1694,11 @@ def testIPv6AddressTooLarge(self): self.assertEqual(ipaddress.ip_address('FFFF::192.0.2.1'), ipaddress.ip_address('FFFF::c000:201')) + self.assertEqual(ipaddress.ip_address('0000:0000:0000:0000:0000:FFFF:192.168.255.255'), + ipaddress.ip_address('::ffff:c0a8:ffff')) + self.assertEqual(ipaddress.ip_address('FFFF:0000:0000:0000:0000:0000:192.168.255.255'), + ipaddress.ip_address('ffff::c0a8:ffff')) + def testIPVersion(self): self.assertEqual(self.ipv4_address.version, 4) self.assertEqual(self.ipv6_address.version, 6) @@ -2065,6 +2072,11 @@ def testCompressIPv6Address(self): '::7:6:5:4:3:2:0': '0:7:6:5:4:3:2:0/128', '7:6:5:4:3:2:1::': '7:6:5:4:3:2:1:0/128', '0:6:5:4:3:2:1::': '0:6:5:4:3:2:1:0/128', + '0000:0000:0000:0000:0000:0000:255.255.255.255': '::ffff:ffff/128', + # 3.7 prints IPv4-mapped addresses in hex (gh-87799 is not backported). + '0000:0000:0000:0000:0000:ffff:255.255.255.255': '::ffff:ffff:ffff/128', + 'ffff:ffff:ffff:ffff:ffff:ffff:255.255.255.255': + 'ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff/128', } for uncompressed, compressed in list(test_addresses.items()): self.assertEqual(compressed, str(ipaddress.IPv6Interface( diff --git a/Misc/NEWS.d/next/Library/2025-05-28-15-53-27.gh-issue-128840.Nur2pB.rst b/Misc/NEWS.d/next/Library/2025-05-28-15-53-27.gh-issue-128840.Nur2pB.rst new file mode 100644 index 000000000000000..faff433aa4b86e3 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-05-28-15-53-27.gh-issue-128840.Nur2pB.rst @@ -0,0 +1 @@ +Fix parsing long IPv6 addresses with embedded IPv4 address. From f0f9c5bcf7a15120f23802e854236f10aa5be6f6 Mon Sep 17 00:00:00 2001 From: "T. Wouters" Date: Thu, 29 Oct 2026 09:52:35 -0500 Subject: [PATCH 022/129] [3.7] gh-135034: Normalize link targets in tarfile, add `os.path.realpath(strict='allow_missing')` (GH-135037) (GH-135084) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Addresses CVEs 2024-12718, 2025-4138, 2025-4330, and 2025-4517. (cherry picked from commit 3612d8f51741b11f36f8fb0494d79086bac9390a) Co-authored-by: Łukasz Langa Co-authored-by: Petr Viktorin Co-authored-by: Seth Michael Larson Co-authored-by: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Co-authored-by: Serhiy Storchaka 3.7.17.x: hand-ported (the 3.9 commit relies on the strict parameter of os.path.realpath, which 3.9 got from bpo-43757 / GH-135035). - posixpath.realpath gains strict=False|True|ALLOW_MISSING (3.9.25 code). - ntpath.realpath() gains the strict keyword here; it only starts resolving links on Windows in the later bpo-9949 commit. - pathlib is left alone (bpo-43757's pathlib change is not needed here). - Tests: test_tarfile changes taken; 3.10+ syntax (parenthesized with, f'{x=}', BaseException.add_note) rewritten for 3.7. (cherry picked from commit dd8f187d0746da151e0025c51680979ac5b4cfb1) 3.7.17.x: test_posixpath gains the realpath strict/ALLOW_MISSING tests from bpo-43757 and gh-135034. --- Doc/library/tarfile.rst | 20 ++ Lib/genericpath.py | 11 +- Lib/ntpath.py | 7 +- Lib/posixpath.py | 32 +- Lib/tarfile.py | 161 +++++++-- Lib/test/test_posixpath.py | 312 +++++++++++++++-- Lib/test/test_tarfile.py | 316 ++++++++++++++++-- ...-06-02-11-32-23.gh-issue-135034.RLGjbp.rst | 6 + 8 files changed, 769 insertions(+), 96 deletions(-) create mode 100644 Misc/NEWS.d/next/Security/2025-06-02-11-32-23.gh-issue-135034.RLGjbp.rst diff --git a/Doc/library/tarfile.rst b/Doc/library/tarfile.rst index 9c37df44beec204..6f7ed8b351280bf 100644 --- a/Doc/library/tarfile.rst +++ b/Doc/library/tarfile.rst @@ -230,6 +230,15 @@ The :mod:`tarfile` module defines the following exceptions: Raised to refuse extracting a symbolic link pointing outside the destination directory. +.. exception:: LinkFallbackError + + Raised to refuse emulating a link (hard or symbolic) by extracting another + archive member, when that member would be rejected by the filter location. + The exception that was raised to reject the replacement member is available + as :attr:`!BaseException.__context__`. + + .. versionadded:: next + The following constants are available at the module level: @@ -930,6 +939,12 @@ reused in custom filters: Implements the ``'data'`` filter. In addition to what ``tar_filter`` does: + - Normalize link targets (:attr:`TarInfo.linkname`) using + :func:`os.path.normpath`. + Note that this removes internal ``..`` components, which may change the + meaning of the link if the path in :attr:`!TarInfo.linkname` traverses + symbolic links. + - :ref:`Refuse ` to extract links (hard or soft) that link to absolute paths, or ones that link outside the destination. @@ -958,6 +973,10 @@ reused in custom filters: Return the modified ``TarInfo`` member. + .. versionchanged:: next + + Link targets are now normalized. + .. _tarfile-extraction-refuse: @@ -984,6 +1003,7 @@ Here is an incomplete list of things to consider: * Extract to a :func:`new temporary directory ` to prevent e.g. exploiting pre-existing links, and to make it easier to clean up after a failed extraction. +* Disallow symbolic links if you do not need the functionality. * When working with untrusted data, use external (e.g. OS-level) limits on disk, memory and CPU usage. * Check filenames against an allow-list of characters diff --git a/Lib/genericpath.py b/Lib/genericpath.py index 85f4a57582f2a2d..065a3be320466e7 100644 --- a/Lib/genericpath.py +++ b/Lib/genericpath.py @@ -8,7 +8,16 @@ __all__ = ['commonprefix', 'exists', 'getatime', 'getctime', 'getmtime', 'getsize', 'isdir', 'isfile', 'samefile', 'sameopenfile', - 'samestat'] + 'samestat', 'ALLOW_MISSING'] + +# A singleton with a true boolean value. +@object.__new__ +class ALLOW_MISSING: + """Special value for use in realpath().""" + def __repr__(self): + return 'os.path.ALLOW_MISSING' + def __reduce__(self): + return self.__class__.__name__ # Does a path exist? diff --git a/Lib/ntpath.py b/Lib/ntpath.py index 3c820b5d0eb791c..bcea255e71d7f40 100644 --- a/Lib/ntpath.py +++ b/Lib/ntpath.py @@ -29,7 +29,7 @@ "ismount", "expanduser","expandvars","normpath","abspath", "curdir","pardir","sep","pathsep","defpath","altsep", "extsep","devnull","realpath","supports_unicode_filenames","relpath", - "samefile", "sameopenfile", "samestat", "commonpath"] + "samefile", "sameopenfile", "samestat", "commonpath", "ALLOW_MISSING"] def _get_bothseps(path): if isinstance(path, bytes): @@ -528,7 +528,10 @@ def abspath(path): return _abspath_fallback(path) # realpath is a no-op on systems without islink support -realpath = abspath +def realpath(path, *, strict=False): + # Symbolic links are not resolved on Windows before 3.8, so *strict* + # (including ALLOW_MISSING) is accepted for compatibility and ignored. + return abspath(path) # Win9x family and earlier have no Unicode filename support. supports_unicode_filenames = (hasattr(sys, "getwindowsversion") and sys.getwindowsversion()[3] >= 2) diff --git a/Lib/posixpath.py b/Lib/posixpath.py index 785aa728b3658c6..bfcf3c862f683d4 100644 --- a/Lib/posixpath.py +++ b/Lib/posixpath.py @@ -35,7 +35,7 @@ "samefile","sameopenfile","samestat", "curdir","pardir","sep","pathsep","defpath","altsep","extsep", "devnull","realpath","supports_unicode_filenames","relpath", - "commonpath"] + "commonpath", "ALLOW_MISSING"] def _get_sep(path): @@ -388,16 +388,16 @@ def abspath(path): # Return a canonical path (i.e. the absolute location of a file on the # filesystem). -def realpath(filename): +def realpath(filename, *, strict=False): """Return the canonical path of the specified filename, eliminating any symbolic links encountered in the path.""" filename = os.fspath(filename) - path, ok = _joinrealpath(filename[:0], filename, {}) + path, ok = _joinrealpath(filename[:0], filename, strict, {}) return abspath(path) # Join two paths, normalizing and eliminating any symbolic links # encountered in the second path. -def _joinrealpath(path, rest, seen): +def _joinrealpath(path, rest, strict, seen): if isinstance(path, bytes): sep = b'/' curdir = b'.' @@ -406,6 +406,12 @@ def _joinrealpath(path, rest, seen): sep = '/' curdir = '.' pardir = '..' + if strict is ALLOW_MISSING: + ignored_error = FileNotFoundError + elif strict: + ignored_error = () + else: + ignored_error = OSError if isabs(rest): rest = rest[1:] @@ -426,7 +432,13 @@ def _joinrealpath(path, rest, seen): path = pardir continue newpath = join(path, name) - if not islink(newpath): + try: + st = os.lstat(newpath) + except ignored_error: + is_link = False + else: + is_link = stat.S_ISLNK(st.st_mode) + if not is_link: path = newpath continue # Resolve the symbolic link @@ -437,10 +449,14 @@ def _joinrealpath(path, rest, seen): # use cached value continue # The symlink is not resolved, so we must have a symlink loop. - # Return already resolved part + rest of the path unchanged. - return join(newpath, rest), False + if strict: + # Raise OSError(errno.ELOOP) + os.stat(newpath) + else: + # Return already resolved part + rest of the path unchanged. + return join(newpath, rest), False seen[newpath] = None # not resolved symlink - path, ok = _joinrealpath(path, os.readlink(newpath), seen) + path, ok = _joinrealpath(path, os.readlink(newpath), strict, seen) if not ok: return join(path, rest), False seen[newpath] = path # resolved symlink diff --git a/Lib/tarfile.py b/Lib/tarfile.py index 41cfd6f2c62119a..ce6737e55286eb9 100755 --- a/Lib/tarfile.py +++ b/Lib/tarfile.py @@ -759,10 +759,22 @@ def __init__(self, tarinfo, path): super().__init__(f'{tarinfo.name!r} would link to {path!r}, ' + 'which is outside the destination') +class LinkFallbackError(FilterError): + def __init__(self, tarinfo, path): + self.tarinfo = tarinfo + self._path = path + super().__init__(f'link {tarinfo.name!r} would be extracted as a ' + + f'copy of {path!r}, which was rejected') + +# Errors caused by filters -- both "fatal" and "non-fatal" -- that +# we consider to be issues with the argument, rather than a bug in the +# filter function +_FILTER_ERRORS = (FilterError, OSError, ExtractError) + def _get_filtered_attrs(member, dest_path, for_data=True): new_attrs = {} name = member.name - dest_path = os.path.realpath(dest_path) + dest_path = os.path.realpath(dest_path, strict=os.path.ALLOW_MISSING) # Strip leading / (tar's directory separator) from filenames. # Include os.sep (target OS directory separator) as well. if name.startswith(('/', os.sep)): @@ -772,7 +784,8 @@ def _get_filtered_attrs(member, dest_path, for_data=True): # For example, 'C:/foo' on Windows. raise AbsolutePathError(member) # Ensure we stay in the destination - target_path = os.path.realpath(os.path.join(dest_path, name)) + target_path = os.path.realpath(os.path.join(dest_path, name), + strict=os.path.ALLOW_MISSING) if os.path.commonpath([target_path, dest_path]) != dest_path: raise OutsideDestinationError(member, target_path) # Limit permissions (no high bits, and go-w) @@ -810,6 +823,9 @@ def _get_filtered_attrs(member, dest_path, for_data=True): if member.islnk() or member.issym(): if os.path.isabs(member.linkname): raise AbsoluteLinkError(member) + normalized = os.path.normpath(member.linkname) + if normalized != member.linkname: + new_attrs['linkname'] = normalized if member.issym(): target_path = os.path.join(dest_path, os.path.dirname(name), @@ -817,7 +833,8 @@ def _get_filtered_attrs(member, dest_path, for_data=True): else: target_path = os.path.join(dest_path, member.linkname) - target_path = os.path.realpath(target_path) + target_path = os.path.realpath(target_path, + strict=os.path.ALLOW_MISSING) if os.path.commonpath([target_path, dest_path]) != dest_path: raise LinkOutsideDestinationError(member, target_path) return new_attrs @@ -2234,30 +2251,58 @@ def extractall(self, path=".", members=None, *, numeric_owner=False, members = self for member in members: - tarinfo = self._get_extract_tarinfo(member, filter_function, path) + tarinfo, unfiltered = self._get_extract_tarinfo( + member, filter_function, path) if tarinfo is None: continue if tarinfo.isdir(): # For directories, delay setting attributes until later, # since permissions can interfere with extraction and # extracting contents can reset mtime. - directories.append(tarinfo) + directories.append(unfiltered) self._extract_one(tarinfo, path, set_attrs=not tarinfo.isdir(), - numeric_owner=numeric_owner) + numeric_owner=numeric_owner, + filter_function=filter_function) # Reverse sort directories. directories.sort(key=lambda a: a.name, reverse=True) + # Set correct owner, mtime and filemode on directories. - for tarinfo in directories: - dirpath = os.path.join(path, tarinfo.name) + for unfiltered in directories: try: + # Need to re-apply any filter, to take the *current* filesystem + # state into account. + try: + tarinfo = filter_function(unfiltered, path) + except _FILTER_ERRORS as exc: + self._log_no_directory_fixup(unfiltered, repr(exc)) + continue + if tarinfo is None: + self._log_no_directory_fixup(unfiltered, + 'excluded by filter') + continue + dirpath = os.path.join(path, tarinfo.name) + try: + lstat = os.lstat(dirpath) + except FileNotFoundError: + self._log_no_directory_fixup(tarinfo, 'missing') + continue + if not stat.S_ISDIR(lstat.st_mode): + # This is no longer a directory; presumably a later + # member overwrote the entry. + self._log_no_directory_fixup(tarinfo, 'not a directory') + continue self.chown(tarinfo, dirpath, numeric_owner=numeric_owner) self.utime(tarinfo, dirpath) self.chmod(tarinfo, dirpath) except ExtractError as e: self._handle_nonfatal_error(e) + def _log_no_directory_fixup(self, member, reason): + self._dbg(2, "tarfile: Not fixing up directory %r (%s)" % + (member.name, reason)) + def extract(self, member, path="", set_attrs=True, *, numeric_owner=False, filter=None): """Extract a member from the archive to the current working directory, @@ -2273,41 +2318,56 @@ def extract(self, member, path="", set_attrs=True, *, numeric_owner=False, String names of common filters are accepted. """ filter_function = self._get_filter_function(filter) - tarinfo = self._get_extract_tarinfo(member, filter_function, path) + tarinfo, unfiltered = self._get_extract_tarinfo( + member, filter_function, path) if tarinfo is not None: self._extract_one(tarinfo, path, set_attrs, numeric_owner) def _get_extract_tarinfo(self, member, filter_function, path): - """Get filtered TarInfo (or None) from member, which might be a str""" + """Get (filtered, unfiltered) TarInfos from *member* + + *member* might be a string. + + Return (None, None) if not found. + """ + if isinstance(member, str): - tarinfo = self.getmember(member) + unfiltered = self.getmember(member) else: - tarinfo = member + unfiltered = member - unfiltered = tarinfo + filtered = None try: - tarinfo = filter_function(tarinfo, path) + filtered = filter_function(unfiltered, path) except (OSError, FilterError) as e: self._handle_fatal_error(e) except ExtractError as e: self._handle_nonfatal_error(e) - if tarinfo is None: + if filtered is None: self._dbg(2, "tarfile: Excluded %r" % unfiltered.name) - return None + return None, None + # Prepare the link target for makelink(). - if tarinfo.islnk(): - tarinfo = copy.copy(tarinfo) - tarinfo._link_target = os.path.join(path, tarinfo.linkname) - return tarinfo + if filtered.islnk(): + filtered = copy.copy(filtered) + filtered._link_target = os.path.join(path, filtered.linkname) + return filtered, unfiltered - def _extract_one(self, tarinfo, path, set_attrs, numeric_owner): - """Extract from filtered tarinfo to disk""" + def _extract_one(self, tarinfo, path, set_attrs, numeric_owner, + filter_function=None): + """Extract from filtered tarinfo to disk. + + filter_function is only used when extracting a *different* + member (e.g. as fallback to creating a symlink) + """ self._check("r") try: self._extract_member(tarinfo, os.path.join(path, tarinfo.name), set_attrs=set_attrs, - numeric_owner=numeric_owner) + numeric_owner=numeric_owner, + filter_function=filter_function, + extraction_root=path) except OSError as e: self._handle_fatal_error(e) except ExtractError as e: @@ -2364,9 +2424,13 @@ def extractfile(self, member): return None def _extract_member(self, tarinfo, targetpath, set_attrs=True, - numeric_owner=False): - """Extract the TarInfo object tarinfo to a physical + numeric_owner=False, *, filter_function=None, + extraction_root=None): + """Extract the filtered TarInfo object tarinfo to a physical file called targetpath. + + filter_function is only used when extracting a *different* + member (e.g. as fallback to creating a symlink) """ # Fetch the TarInfo object for the given name # and build the destination pathname, replacing @@ -2395,7 +2459,10 @@ def _extract_member(self, tarinfo, targetpath, set_attrs=True, elif tarinfo.ischr() or tarinfo.isblk(): self.makedev(tarinfo, targetpath) elif tarinfo.islnk() or tarinfo.issym(): - self.makelink(tarinfo, targetpath) + self.makelink_with_filter( + tarinfo, targetpath, + filter_function=filter_function, + extraction_root=extraction_root) elif tarinfo.type not in SUPPORTED_TYPES: self.makeunknown(tarinfo, targetpath) else: @@ -2477,26 +2544,54 @@ def makedev(self, tarinfo, targetpath): os.makedev(tarinfo.devmajor, tarinfo.devminor)) def makelink(self, tarinfo, targetpath): + return self.makelink_with_filter(tarinfo, targetpath, None, None) + + def makelink_with_filter(self, tarinfo, targetpath, + filter_function, extraction_root): """Make a (symbolic) link called targetpath. If it cannot be created (platform limitation), we try to make a copy of the referenced file instead of a link. + + filter_function is only used when extracting a *different* + member (e.g. as fallback to creating a link). """ + keyerror_to_extracterror = False try: # For systems that support symbolic and hard links. if tarinfo.issym(): os.symlink(tarinfo.linkname, targetpath) + return else: if os.path.exists(tarinfo._link_target): os.link(tarinfo._link_target, targetpath) - else: - self._extract_member(self._find_link_target(tarinfo), - targetpath) + return except symlink_exception: + keyerror_to_extracterror = True + + try: + unfiltered = self._find_link_target(tarinfo) + except KeyError: + if keyerror_to_extracterror: + raise ExtractError( + "unable to resolve link inside archive") + else: + raise + + if filter_function is None: + filtered = unfiltered + else: + if extraction_root is None: + raise ExtractError( + "makelink_with_filter: if filter_function is not None, " + + "extraction_root must also not be None") try: - self._extract_member(self._find_link_target(tarinfo), - targetpath) - except KeyError: - raise ExtractError("unable to resolve link inside archive") + filtered = filter_function(unfiltered, extraction_root) + except _FILTER_ERRORS as cause: + raise LinkFallbackError(tarinfo, unfiltered.name) from cause + if filtered is not None: + self._extract_member(filtered, targetpath, + filter_function=filter_function, + extraction_root=extraction_root) def chown(self, tarinfo, targetpath, numeric_owner): """Set owner of targetpath according to tarinfo. If numeric_owner diff --git a/Lib/test/test_posixpath.py b/Lib/test/test_posixpath.py index e73b31cb648b227..dc615e0e8edc5ca 100644 --- a/Lib/test/test_posixpath.py +++ b/Lib/test/test_posixpath.py @@ -1,8 +1,10 @@ import os +import sys import posixpath import unittest import warnings -from posixpath import realpath, abspath, dirname, basename +from functools import partial +from posixpath import realpath, abspath, dirname, basename, ALLOW_MISSING from test import support, test_genericpath from test.support import FakePath from unittest import mock @@ -33,6 +35,26 @@ def safe_rmdir(dirname): except OSError: pass +def _parameterize(*parameters): + """Simplistic decorator to parametrize a test + + Runs the decorated test multiple times in subTest, with a value from + 'parameters' passed as an extra positional argument. + Does *not* call doCleanups() after each run. + + Not for general use. Intended to avoid indenting for easier backports. + + See https://discuss.python.org/t/91827 for discussing generalizations. + """ + def _parametrize_decorator(func): + def _parameterized(self, *args, **kwargs): + for parameter in parameters: + with self.subTest(parameter): + func(self, *args, parameter, **kwargs) + return _parameterized + return _parametrize_decorator + + class PosixPathTest(unittest.TestCase): def setUp(self): @@ -308,52 +330,167 @@ def test_normpath(self): b"/foo/bar") @skip_if_ABSTFN_contains_backslash - def test_realpath_curdir(self): - self.assertEqual(realpath('.'), os.getcwd()) - self.assertEqual(realpath('./.'), os.getcwd()) - self.assertEqual(realpath('/'.join(['.'] * 100)), os.getcwd()) + @_parameterize({}, {'strict': True}, {'strict': ALLOW_MISSING}) + def test_realpath_curdir(self, kwargs): + self.assertEqual(realpath('.', **kwargs), os.getcwd()) + self.assertEqual(realpath('./.', **kwargs), os.getcwd()) + self.assertEqual(realpath('/'.join(['.'] * 100), **kwargs), os.getcwd()) - self.assertEqual(realpath(b'.'), os.getcwdb()) - self.assertEqual(realpath(b'./.'), os.getcwdb()) - self.assertEqual(realpath(b'/'.join([b'.'] * 100)), os.getcwdb()) + self.assertEqual(realpath(b'.', **kwargs), os.getcwdb()) + self.assertEqual(realpath(b'./.', **kwargs), os.getcwdb()) + self.assertEqual(realpath(b'/'.join([b'.'] * 100), **kwargs), os.getcwdb()) @skip_if_ABSTFN_contains_backslash - def test_realpath_pardir(self): - self.assertEqual(realpath('..'), dirname(os.getcwd())) - self.assertEqual(realpath('../..'), dirname(dirname(os.getcwd()))) - self.assertEqual(realpath('/'.join(['..'] * 100)), '/') + @_parameterize({}, {'strict': True}, {'strict': ALLOW_MISSING}) + def test_realpath_pardir(self, kwargs): + self.assertEqual(realpath('..', **kwargs), dirname(os.getcwd())) + self.assertEqual(realpath('../..', **kwargs), dirname(dirname(os.getcwd()))) + self.assertEqual(realpath('/'.join(['..'] * 100), **kwargs), '/') - self.assertEqual(realpath(b'..'), dirname(os.getcwdb())) - self.assertEqual(realpath(b'../..'), dirname(dirname(os.getcwdb()))) - self.assertEqual(realpath(b'/'.join([b'..'] * 100)), b'/') + self.assertEqual(realpath(b'..', **kwargs), dirname(os.getcwdb())) + self.assertEqual(realpath(b'../..', **kwargs), dirname(dirname(os.getcwdb()))) + self.assertEqual(realpath(b'/'.join([b'..'] * 100), **kwargs), b'/') @unittest.skipUnless(hasattr(os, "symlink"), "Missing symlink implementation") @skip_if_ABSTFN_contains_backslash - def test_realpath_basic(self): + @_parameterize({}, {'strict': ALLOW_MISSING}) + def test_realpath_basic(self, kwargs): # Basic operation. try: os.symlink(ABSTFN+"1", ABSTFN) - self.assertEqual(realpath(ABSTFN), ABSTFN+"1") + self.assertEqual(realpath(ABSTFN, **kwargs), ABSTFN+"1") finally: support.unlink(ABSTFN) @unittest.skipUnless(hasattr(os, "symlink"), "Missing symlink implementation") @skip_if_ABSTFN_contains_backslash - def test_realpath_relative(self): + def test_realpath_strict(self): + # Bug #43757: raise FileNotFoundError in strict mode if we encounter + # a path that does not exist. + try: + os.symlink(ABSTFN+"1", ABSTFN) + self.assertRaises(FileNotFoundError, realpath, ABSTFN, strict=True) + self.assertRaises(FileNotFoundError, realpath, ABSTFN + "2", strict=True) + finally: + support.unlink(ABSTFN) + + def test_realpath_invalid_paths(self): + path = '/\x00' + self.assertRaises(ValueError, realpath, path, strict=False) + self.assertRaises(ValueError, realpath, path, strict=True) + self.assertRaises(ValueError, realpath, path, strict=ALLOW_MISSING) + path = b'/\x00' + self.assertRaises(ValueError, realpath, path, strict=False) + self.assertRaises(ValueError, realpath, path, strict=True) + self.assertRaises(ValueError, realpath, path, strict=ALLOW_MISSING) + path = '/nonexistent/x\x00' + self.assertRaises(ValueError, realpath, path, strict=False) + self.assertRaises(FileNotFoundError, realpath, path, strict=True) + self.assertRaises(ValueError, realpath, path, strict=ALLOW_MISSING) + path = b'/nonexistent/x\x00' + self.assertRaises(ValueError, realpath, path, strict=False) + self.assertRaises(FileNotFoundError, realpath, path, strict=True) + self.assertRaises(ValueError, realpath, path, strict=ALLOW_MISSING) + path = '/\x00/..' + self.assertRaises(ValueError, realpath, path, strict=False) + self.assertRaises(ValueError, realpath, path, strict=True) + self.assertRaises(ValueError, realpath, path, strict=ALLOW_MISSING) + path = b'/\x00/..' + self.assertRaises(ValueError, realpath, path, strict=False) + self.assertRaises(ValueError, realpath, path, strict=True) + self.assertRaises(ValueError, realpath, path, strict=ALLOW_MISSING) + + path = '/nonexistent/x\x00/..' + self.assertRaises(ValueError, realpath, path, strict=False) + self.assertRaises(FileNotFoundError, realpath, path, strict=True) + self.assertRaises(ValueError, realpath, path, strict=ALLOW_MISSING) + path = b'/nonexistent/x\x00/..' + self.assertRaises(ValueError, realpath, path, strict=False) + self.assertRaises(FileNotFoundError, realpath, path, strict=True) + self.assertRaises(ValueError, realpath, path, strict=ALLOW_MISSING) + + path = '/\udfff' + if sys.platform == 'win32': + self.assertEqual(realpath(path, strict=False), path) + self.assertRaises(FileNotFoundError, realpath, path, strict=True) + self.assertEqual(realpath(path, strict=ALLOW_MISSING), path) + else: + self.assertRaises(UnicodeEncodeError, realpath, path, strict=False) + self.assertRaises(UnicodeEncodeError, realpath, path, strict=True) + self.assertRaises(UnicodeEncodeError, realpath, path, strict=ALLOW_MISSING) + path = '/nonexistent/\udfff' + if sys.platform == 'win32': + self.assertEqual(realpath(path, strict=False), path) + self.assertEqual(realpath(path, strict=ALLOW_MISSING), path) + else: + self.assertRaises(UnicodeEncodeError, realpath, path, strict=False) + self.assertRaises(UnicodeEncodeError, realpath, path, strict=ALLOW_MISSING) + self.assertRaises(FileNotFoundError, realpath, path, strict=True) + path = '/\udfff/..' + if sys.platform == 'win32': + self.assertEqual(realpath(path, strict=False), '/') + self.assertRaises(FileNotFoundError, realpath, path, strict=True) + self.assertEqual(realpath(path, strict=ALLOW_MISSING), '/') + else: + self.assertRaises(UnicodeEncodeError, realpath, path, strict=False) + self.assertRaises(UnicodeEncodeError, realpath, path, strict=True) + self.assertRaises(UnicodeEncodeError, realpath, path, strict=ALLOW_MISSING) + path = '/nonexistent/\udfff/..' + if sys.platform == 'win32': + self.assertEqual(realpath(path, strict=False), '/nonexistent') + self.assertEqual(realpath(path, strict=ALLOW_MISSING), '/nonexistent') + else: + self.assertRaises(UnicodeEncodeError, realpath, path, strict=False) + self.assertRaises(UnicodeEncodeError, realpath, path, strict=ALLOW_MISSING) + self.assertRaises(FileNotFoundError, realpath, path, strict=True) + + path = b'/\xff' + if sys.platform == 'win32': + self.assertRaises(UnicodeDecodeError, realpath, path, strict=False) + self.assertRaises(UnicodeDecodeError, realpath, path, strict=True) + self.assertRaises(UnicodeDecodeError, realpath, path, strict=ALLOW_MISSING) + else: + self.assertEqual(realpath(path, strict=False), path) + self.assertRaises(FileNotFoundError, realpath, path, strict=True) + self.assertEqual(realpath(path, strict=ALLOW_MISSING), path) + path = b'/nonexistent/\xff' + if sys.platform == 'win32': + self.assertRaises(UnicodeDecodeError, realpath, path, strict=False) + self.assertRaises(UnicodeDecodeError, realpath, path, strict=ALLOW_MISSING) + else: + self.assertEqual(realpath(path, strict=False), path) + self.assertRaises(FileNotFoundError, realpath, path, strict=True) + + @unittest.skipUnless(hasattr(os, "symlink"), + "Missing symlink implementation") + @skip_if_ABSTFN_contains_backslash + @_parameterize({}, {'strict': ALLOW_MISSING}) + def test_realpath_relative(self, kwargs): try: os.symlink(posixpath.relpath(ABSTFN+"1"), ABSTFN) - self.assertEqual(realpath(ABSTFN), ABSTFN+"1") + self.assertEqual(realpath(ABSTFN, **kwargs), ABSTFN+"1") finally: support.unlink(ABSTFN) @unittest.skipUnless(hasattr(os, "symlink"), "Missing symlink implementation") @skip_if_ABSTFN_contains_backslash + @_parameterize({}, {'strict': ALLOW_MISSING}) + def test_realpath_missing_pardir(self, kwargs): + try: + os.symlink(support.TESTFN + "1", support.TESTFN) + self.assertEqual( + realpath("nonexistent/../" + support.TESTFN, **kwargs), ABSTFN + "1") + finally: + support.unlink(support.TESTFN) + + @support.skip_unless_symlink + @skip_if_ABSTFN_contains_backslash def test_realpath_symlink_loops(self): # Bug #930024, return the path unchanged if we get into an infinite - # symlink loop. + # symlink loop in non-strict mode (default). try: os.symlink(ABSTFN, ABSTFN) self.assertEqual(realpath(ABSTFN), ABSTFN) @@ -393,13 +530,57 @@ def test_realpath_symlink_loops(self): @unittest.skipUnless(hasattr(os, "symlink"), "Missing symlink implementation") @skip_if_ABSTFN_contains_backslash - def test_realpath_repeated_indirect_symlinks(self): + @_parameterize({'strict': True}, {'strict': ALLOW_MISSING}) + def test_realpath_symlink_loops_strict(self, kwargs): + # Bug #43757, raise OSError if we get into an infinite symlink loop in + # the strict modes. + try: + os.symlink(ABSTFN, ABSTFN) + self.assertRaises(OSError, realpath, ABSTFN, **kwargs) + + os.symlink(ABSTFN+"1", ABSTFN+"2") + os.symlink(ABSTFN+"2", ABSTFN+"1") + self.assertRaises(OSError, realpath, ABSTFN+"1", **kwargs) + self.assertRaises(OSError, realpath, ABSTFN+"2", **kwargs) + + self.assertRaises(OSError, realpath, ABSTFN+"1/x", **kwargs) + self.assertRaises(OSError, realpath, ABSTFN+"1/..", **kwargs) + self.assertRaises(OSError, realpath, ABSTFN+"1/../x", **kwargs) + os.symlink(ABSTFN+"x", ABSTFN+"y") + self.assertRaises(OSError, realpath, + ABSTFN+"1/../" + basename(ABSTFN) + "y", **kwargs) + self.assertRaises(OSError, realpath, + ABSTFN+"1/../" + basename(ABSTFN) + "1", **kwargs) + + os.symlink(basename(ABSTFN) + "a/b", ABSTFN+"a") + self.assertRaises(OSError, realpath, ABSTFN+"a", **kwargs) + + os.symlink("../" + basename(dirname(ABSTFN)) + "/" + + basename(ABSTFN) + "c", ABSTFN+"c") + self.assertRaises(OSError, realpath, ABSTFN+"c", **kwargs) + + # Test using relative path as well. + with support.change_cwd(dirname(ABSTFN)): + self.assertRaises(OSError, realpath, basename(ABSTFN), **kwargs) + finally: + support.unlink(ABSTFN) + support.unlink(ABSTFN+"1") + support.unlink(ABSTFN+"2") + support.unlink(ABSTFN+"y") + support.unlink(ABSTFN+"c") + support.unlink(ABSTFN+"a") + + @unittest.skipUnless(hasattr(os, "symlink"), + "Missing symlink implementation") + @skip_if_ABSTFN_contains_backslash + @_parameterize({}, {'strict': True}, {'strict': ALLOW_MISSING}) + def test_realpath_repeated_indirect_symlinks(self, kwargs): # Issue #6975. try: os.mkdir(ABSTFN) os.symlink('../' + basename(ABSTFN), ABSTFN + '/self') os.symlink('self/self/self', ABSTFN + '/link') - self.assertEqual(realpath(ABSTFN + '/link'), ABSTFN) + self.assertEqual(realpath(ABSTFN + '/link', **kwargs), ABSTFN) finally: support.unlink(ABSTFN + '/self') support.unlink(ABSTFN + '/link') @@ -408,14 +589,15 @@ def test_realpath_repeated_indirect_symlinks(self): @unittest.skipUnless(hasattr(os, "symlink"), "Missing symlink implementation") @skip_if_ABSTFN_contains_backslash - def test_realpath_deep_recursion(self): + @_parameterize({}, {'strict': True}, {'strict': ALLOW_MISSING}) + def test_realpath_deep_recursion(self, kwargs): depth = 10 try: os.mkdir(ABSTFN) for i in range(depth): os.symlink('/'.join(['%d' % i] * 10), ABSTFN + '/%d' % (i + 1)) os.symlink('.', ABSTFN + '/0') - self.assertEqual(realpath(ABSTFN + '/%d' % depth), ABSTFN) + self.assertEqual(realpath(ABSTFN + '/%d' % depth, **kwargs), ABSTFN) # Test using relative path as well. with support.change_cwd(ABSTFN): @@ -428,7 +610,8 @@ def test_realpath_deep_recursion(self): @unittest.skipUnless(hasattr(os, "symlink"), "Missing symlink implementation") @skip_if_ABSTFN_contains_backslash - def test_realpath_resolve_parents(self): + @_parameterize({}, {'strict': ALLOW_MISSING}) + def test_realpath_resolve_parents(self, kwargs): # We also need to resolve any symlinks in the parents of a relative # path passed to realpath. E.g.: current working directory is # /usr/doc with 'doc' being a symlink to /usr/share/doc. We call @@ -439,7 +622,8 @@ def test_realpath_resolve_parents(self): os.symlink(ABSTFN + "/y", ABSTFN + "/k") with support.change_cwd(ABSTFN + "/k"): - self.assertEqual(realpath("a"), ABSTFN + "/y/a") + self.assertEqual(realpath("a", **kwargs), + ABSTFN + "/y/a") finally: support.unlink(ABSTFN + "/k") safe_rmdir(ABSTFN + "/y") @@ -448,7 +632,8 @@ def test_realpath_resolve_parents(self): @unittest.skipUnless(hasattr(os, "symlink"), "Missing symlink implementation") @skip_if_ABSTFN_contains_backslash - def test_realpath_resolve_before_normalizing(self): + @_parameterize({}, {'strict': True}, {'strict': ALLOW_MISSING}) + def test_realpath_resolve_before_normalizing(self, kwargs): # Bug #990669: Symbolic links should be resolved before we # normalize the path. E.g.: if we have directories 'a', 'k' and 'y' # in the following hierarchy: @@ -463,10 +648,10 @@ def test_realpath_resolve_before_normalizing(self): os.symlink(ABSTFN + "/k/y", ABSTFN + "/link-y") # Absolute path. - self.assertEqual(realpath(ABSTFN + "/link-y/.."), ABSTFN + "/k") + self.assertEqual(realpath(ABSTFN + "/link-y/..", **kwargs), ABSTFN + "/k") # Relative path. with support.change_cwd(dirname(ABSTFN)): - self.assertEqual(realpath(basename(ABSTFN) + "/link-y/.."), + self.assertEqual(realpath(basename(ABSTFN) + "/link-y/..", **kwargs), ABSTFN + "/k") finally: support.unlink(ABSTFN + "/link-y") @@ -477,7 +662,8 @@ def test_realpath_resolve_before_normalizing(self): @unittest.skipUnless(hasattr(os, "symlink"), "Missing symlink implementation") @skip_if_ABSTFN_contains_backslash - def test_realpath_resolve_first(self): + @_parameterize({}, {'strict': True}, {'strict': ALLOW_MISSING}) + def test_realpath_resolve_first(self, kwargs): # Bug #1213894: The first component of the path, if not absolute, # must be resolved too. @@ -487,13 +673,70 @@ def test_realpath_resolve_first(self): os.symlink(ABSTFN, ABSTFN + "link") with support.change_cwd(dirname(ABSTFN)): base = basename(ABSTFN) - self.assertEqual(realpath(base + "link"), ABSTFN) - self.assertEqual(realpath(base + "link/k"), ABSTFN + "/k") + self.assertEqual(realpath(base + "link", **kwargs), ABSTFN) + self.assertEqual(realpath(base + "link/k", **kwargs), ABSTFN + "/k") finally: support.unlink(ABSTFN + "link") safe_rmdir(ABSTFN + "/k") safe_rmdir(ABSTFN) + @support.skip_unless_symlink + @skip_if_ABSTFN_contains_backslash + @unittest.skipIf(os.chmod not in os.supports_follow_symlinks, "Can't set symlink permissions") + @unittest.skipIf(sys.platform != "darwin", "only macOS requires read permission to readlink()") + @_parameterize({'strict': True}, {'strict': ALLOW_MISSING}) + def test_realpath_unreadable_symlink_strict(self, kwargs): + try: + os.symlink(ABSTFN+"1", ABSTFN) + os.chmod(ABSTFN, 0o000, follow_symlinks=False) + with self.assertRaises(PermissionError): + realpath(ABSTFN, **kwargs) + with self.assertRaises(PermissionError): + realpath(ABSTFN + '/foo', **kwargs), + with self.assertRaises(PermissionError): + realpath(ABSTFN + '/../foo', **kwargs) + with self.assertRaises(PermissionError): + realpath(ABSTFN + '/foo/..', **kwargs) + finally: + os.chmod(ABSTFN, 0o755, follow_symlinks=False) + os.unlink(ABSTFN) + + @skip_if_ABSTFN_contains_backslash + @support.skip_unless_symlink + def test_realpath_unreadable_directory(self): + try: + os.mkdir(ABSTFN) + os.mkdir(ABSTFN + '/k') + os.chmod(ABSTFN, 0o000) + self.assertEqual(realpath(ABSTFN, strict=False), ABSTFN) + self.assertEqual(realpath(ABSTFN, strict=True), ABSTFN) + self.assertEqual(realpath(ABSTFN, strict=ALLOW_MISSING), ABSTFN) + + try: + os.stat(ABSTFN) + except PermissionError: + pass + else: + self.skipTest('Cannot block permissions') + + self.assertEqual(realpath(ABSTFN + '/k', strict=False), + ABSTFN + '/k') + self.assertRaises(PermissionError, realpath, ABSTFN + '/k', + strict=True) + self.assertRaises(PermissionError, realpath, ABSTFN + '/k', + strict=ALLOW_MISSING) + + self.assertEqual(realpath(ABSTFN + '/missing', strict=False), + ABSTFN + '/missing') + self.assertRaises(PermissionError, realpath, ABSTFN + '/missing', + strict=True) + self.assertRaises(PermissionError, realpath, ABSTFN + '/missing', + strict=ALLOW_MISSING) + finally: + os.chmod(ABSTFN, 0o755) + safe_rmdir(ABSTFN + '/k') + safe_rmdir(ABSTFN) + def test_relpath(self): (real_getcwd, os.getcwd) = (os.getcwd, lambda: r"/home/user/bar") try: @@ -670,9 +913,12 @@ def test_path_normpath(self): def test_path_abspath(self): self.assertPathEqual(self.path.abspath) - def test_path_realpath(self): + @_parameterize({}, {'strict': True}, {'strict': ALLOW_MISSING}) + def test_path_realpath(self, kwargs): self.assertPathEqual(self.path.realpath) + self.assertPathEqual(partial(self.path.realpath, **kwargs)) + def test_path_relpath(self): self.assertPathEqual(self.path.relpath) diff --git a/Lib/test/test_tarfile.py b/Lib/test/test_tarfile.py index 13eec2f276a3081..8ff427e293f8c54 100644 --- a/Lib/test/test_tarfile.py +++ b/Lib/test/test_tarfile.py @@ -1,4 +1,5 @@ import sys +import errno import os import io from hashlib import md5 @@ -2245,9 +2246,35 @@ def test__all__(self): 'tar_filter', 'FilterError', 'AbsoluteLinkError', 'OutsideDestinationError', 'SpecialFileError', 'AbsolutePathError', 'LinkOutsideDestinationError', + 'LinkFallbackError', } support.check__all__(self, tarfile, blacklist=blacklist) + @unittest.skipUnless(support.can_symlink(), 'requires symlink support') + @unittest.skipUnless(hasattr(os, 'chmod'), "missing os.chmod") + @unittest.mock.patch('os.chmod') + def test_deferred_directory_attributes_update(self, mock_chmod): + # Regression test for gh-127987: setting attributes on arbitrary files + tempdir = os.path.join(TEMPDIR, 'test127987') + def mock_chmod_side_effect(path, mode, **kwargs): + target_path = os.path.realpath(path) + if os.path.commonpath([target_path, tempdir]) != tempdir: + raise Exception("should not try to chmod anything outside the destination", target_path) + mock_chmod.side_effect = mock_chmod_side_effect + + outside_tree_dir = os.path.join(TEMPDIR, 'outside_tree_dir') + with ArchiveMaker() as arc: + arc.add('x', symlink_to='.') + arc.add('x', type=tarfile.DIRTYPE, mode='?rwsrwsrwt') + arc.add('x', symlink_to=outside_tree_dir) + + os.makedirs(outside_tree_dir) + try: + arc.open().extractall(path=tempdir, filter='tar') + finally: + support.rmtree(outside_tree_dir) + support.rmtree(tempdir) + class CommandLineTest(unittest.TestCase): @@ -2800,6 +2827,10 @@ def check_files_present(self, directory): got_paths = set( p.relative_to(directory) for p in pathlib.Path(directory).glob('**/*')) + if self.extraction_filter == 'data': + # The 'data' filter is expected to reject special files + for path in 'ustar/fifotype', 'ustar/blktype', 'ustar/chrtype': + got_paths.discard(pathlib.Path(path)) self.assertEqual(self.control_paths, got_paths) @contextmanager @@ -3026,12 +3057,28 @@ def __exit__(self, *exc): self.bio = None def add(self, name, *, type=None, symlink_to=None, hardlink_to=None, - mode=None, size=None, **kwargs): - """Add a member to the test archive. Call within `with`.""" + mode=None, size=None, content=None, **kwargs): + """Add a member to the test archive. Call within `with`. + + Provides many shortcuts: + - default `type` is based on symlink_to, hardlink_to, and trailing `/` + in name (which is stripped) + - size & content defaults are based on each other + - content can be str or bytes + - mode should be textual ('-rwxrwxrwx') + + (add more! this is unstable internal test-only API) + """ name = str(name) tarinfo = tarfile.TarInfo(name).replace(**kwargs) + if content is not None: + if isinstance(content, str): + content = content.encode() + size = len(content) if size is not None: tarinfo.size = size + if content is None: + content = bytes(tarinfo.size) if mode: tarinfo.mode = _filemode_to_int(mode) if symlink_to is not None: @@ -3045,7 +3092,7 @@ def add(self, name, *, type=None, symlink_to=None, hardlink_to=None, if type is not None: tarinfo.type = type if tarinfo.isreg(): - fileobj = io.BytesIO(bytes(tarinfo.size)) + fileobj = io.BytesIO(content) else: fileobj = None self.tar_w.addfile(tarinfo, fileobj) @@ -3067,7 +3114,7 @@ class TestExtractionFilters(unittest.TestCase): destdir = outerdir / 'dest' @contextmanager - def check_context(self, tar, filter): + def check_context(self, tar, filter, *, check_flag=True, ignored_trees=()): """Extracts `tar` to `self.destdir` and allows checking the result If an error occurs, it must be checked using `expect_exception` @@ -3076,15 +3123,29 @@ def check_context(self, tar, filter): except the destination directory itself and parent directories of other files. When checking directories, do so before their contents. + + A file called 'flag' is made in outerdir (i.e. outside destdir) + before extraction; it should not be altered nor should its contents + be read/copied. + + *ignored_trees* is a set of directories to remove (including their + contents) right after the archive is extracted. It is a workaround + for Path.glob() failing to get all files in Python 3.10 and below. """ with support.temp_dir(self.outerdir): + flag_path = self.outerdir / 'flag' + flag_path.write_text('capture me') try: tar.extractall(self.destdir, filter=filter) except Exception as exc: self.raised_exception = exc + self.reraise_exception = True self.expected_paths = set() else: + for ignored_tree in ignored_trees: + support.rmtree((self.destdir / ignored_tree).resolve()) self.raised_exception = None + self.reraise_exception = False try: self.expected_paths = set(self.outerdir.glob('**/*')) except OSError as err: @@ -3096,16 +3157,21 @@ def check_context(self, tar, filter): else: raise self.expected_paths.discard(self.destdir) + self.expected_paths.discard(flag_path) try: - yield + yield self finally: tar.close() - if self.raised_exception: + if self.reraise_exception: raise self.raised_exception self.assertEqual(self.expected_paths, set()) + if check_flag: + self.assertEqual(flag_path.read_text(), 'capture me') + else: + assert filter == 'fully_trusted' def expect_file(self, name, type=None, symlink_to=None, mode=None, - size=None): + size=None, content=None): """Check a single file. See check_context.""" if self.raised_exception: raise self.raised_exception @@ -3115,7 +3181,7 @@ def expect_file(self, name, type=None, symlink_to=None, mode=None, self.expected_paths.remove(path) # When checking mode, ignore Windows (which can only set user read and - # user write bits). Newer versions of Python use `os_helper.can_chmod()` + # user write bits). Newer versions of Python use `support.can_chmod()` # instead of hardcoding Windows. if mode is not None and sys.platform != 'win32': got = stat.filemode(stat.S_IMODE(path.stat().st_mode)) @@ -3129,26 +3195,46 @@ def expect_file(self, name, type=None, symlink_to=None, mode=None, # The symlink might be the same (textually) as what we expect, # but some systems change the link to an equivalent path, so # we fall back to samefile(). - if expected != got: - self.assertTrue(got.samefile(expected)) + try: + if expected != got: + self.assertTrue(got.samefile(expected)) + except Exception as e: + # show both paths, even if `samefile` fails + # (no BaseException.add_note() before 3.11) + raise AssertionError( + 'expected=%r, got=%r' % (expected, got)) from e elif type == tarfile.REGTYPE or type is None: self.assertTrue(path.is_file()) elif type == tarfile.DIRTYPE: self.assertTrue(path.is_dir()) elif type == tarfile.FIFOTYPE: self.assertTrue(path.is_fifo()) + elif type == tarfile.SYMTYPE: + self.assertTrue(path.is_symlink()) else: raise NotImplementedError(type) if size is not None: self.assertEqual(path.stat().st_size, size) + if content is not None: + self.assertEqual(path.read_text(), content) for parent in path.parents: self.expected_paths.discard(parent) + def expect_any_tree(self, name): + """Check a directory; forget about its contents.""" + tree_path = (self.destdir / name).resolve() + self.expect_file(tree_path, type=tarfile.DIRTYPE) + self.expected_paths = { + p for p in self.expected_paths + if tree_path not in p.parents + } + def expect_exception(self, exc_type, message_re='.'): with self.assertRaisesRegex(exc_type, message_re): if self.raised_exception is not None: raise self.raised_exception - self.raised_exception = None + self.reraise_exception = False + return self.raised_exception def test_benign_file(self): with ArchiveMaker() as arc: @@ -3232,6 +3318,78 @@ def test_parent_symlink(self): with self.check_context(arc.open(), 'data'): self.expect_file('parent/evil') + @support.skip_unless_symlink + def test_realpath_limit_attack(self): + # (CVE-2025-4517) + + with ArchiveMaker() as arc: + # populate the symlinks and dirs that expand in os.path.realpath() + # The component length is chosen so that in common cases, the unexpanded + # path fits in PATH_MAX, but it overflows when the final symlink + # is expanded + steps = "abcdefghijklmnop" + if sys.platform == 'win32': + component = 'd' * 25 + elif 'PC_PATH_MAX' in os.pathconf_names: + max_path_len = os.pathconf(self.outerdir.parent, "PC_PATH_MAX") + path_sep_len = 1 + dest_len = len(str(self.destdir)) + path_sep_len + component_len = (max_path_len - dest_len) // (len(steps) + path_sep_len) + component = 'd' * component_len + else: + raise NotImplementedError("Need to guess component length for {sys.platform}") + path = "" + step_path = "" + for i in steps: + arc.add(os.path.join(path, component), type=tarfile.DIRTYPE, + mode='drwxrwxrwx') + arc.add(os.path.join(path, i), symlink_to=component) + path = os.path.join(path, component) + step_path = os.path.join(step_path, i) + # create the final symlink that exceeds PATH_MAX and simply points + # to the top dir. + # this link will never be expanded by + # os.path.realpath(strict=False), nor anything after it. + linkpath = os.path.join(*steps, "l"*254) + parent_segments = [".."] * len(steps) + arc.add(linkpath, symlink_to=os.path.join(*parent_segments)) + # make a symlink outside to keep the tar command happy + arc.add("escape", symlink_to=os.path.join(linkpath, "..")) + # use the symlinks above, that are not checked, to create a hardlink + # to a file outside of the destination path + arc.add("flaglink", hardlink_to=os.path.join("escape", "flag")) + # now that we have the hardlink we can overwrite the file + arc.add("flaglink", content='overwrite') + # we can also create new files as well! + arc.add("escape/newfile", content='new') + + with self.subTest('fully_trusted'), \ + self.check_context(arc.open(), filter='fully_trusted', + check_flag=False, ignored_trees={component}): + if sys.platform == 'win32': + self.expect_exception((FileNotFoundError, FileExistsError)) + elif self.raised_exception: + # Cannot symlink/hardlink: tarfile falls back to getmember() + self.expect_exception(KeyError) + # Otherwise, this block should never enter. + else: + self.expect_file('flaglink', content='overwrite') + self.expect_file('../newfile', content='new') + self.expect_file('escape', type=tarfile.SYMTYPE) + self.expect_file('a', symlink_to=component) + + for filter in 'tar', 'data': + with self.subTest(filter), self.check_context(arc.open(), filter=filter): + exc = self.expect_exception((OSError, KeyError)) + if isinstance(exc, OSError): + if sys.platform == 'win32': + # 3: ERROR_PATH_NOT_FOUND + # 5: ERROR_ACCESS_DENIED + # 206: ERROR_FILENAME_EXCED_RANGE + self.assertIn(exc.winerror, (3, 5, 206)) + else: + self.assertEqual(exc.errno, errno.ENAMETOOLONG) + def test_parent_symlink2(self): # Test interplaying symlinks # Inspired by 'dirsymlink2b' in jwilk/traversal-archives @@ -3448,8 +3606,8 @@ def test_chains(self): arc.add('symlink2', symlink_to=os.path.join( 'linkdir', 'hardlink2')) arc.add('targetdir/target', size=3) - arc.add('linkdir/hardlink', hardlink_to='targetdir/target') - arc.add('linkdir/hardlink2', hardlink_to='linkdir/symlink') + arc.add('linkdir/hardlink', hardlink_to=os.path.join('targetdir', 'target')) + arc.add('linkdir/hardlink2', hardlink_to=os.path.join('linkdir', 'symlink')) for filter in 'tar', 'data', 'fully_trusted': with self.check_context(arc.open(), filter): @@ -3465,6 +3623,126 @@ def test_chains(self): self.expect_file('linkdir/symlink', size=3) self.expect_file('symlink2', size=3) + def test_sneaky_hardlink_fallback(self): + # (CVE-2025-4330) + # Test that when hardlink extraction falls back to extracting members + # from the archive, the extracted member is (re-)filtered. + with ArchiveMaker() as arc: + # Create a directory structure so the c/escape symlink stays + # inside the path + arc.add("a/t/dummy") + # Create b/ directory + arc.add("b/") + # Point "c" to the bottom of the tree in "a" + arc.add("c", symlink_to=os.path.join("a", "t")) + # link to non-existant location under "a" + arc.add("c/escape", symlink_to=os.path.join("..", "..", + "link_here")) + # Move "c" to point to "b" ("c/escape" no longer exists) + arc.add("c", symlink_to="b") + # Attempt to create a hard link to "c/escape". Since it doesn't + # exist it will attempt to extract "cescape" but at "boom". + arc.add("boom", hardlink_to=os.path.join("c", "escape")) + + with self.check_context(arc.open(), 'data'): + if not support.can_symlink(): + # When 'c/escape' is extracted, 'c' is a regular + # directory, and 'c/escape' *would* point outside + # the destination if symlinks were allowed. + self.expect_exception( + tarfile.LinkOutsideDestinationError) + elif sys.platform == "win32": + # On Windows, 'c/escape' points outside the destination + self.expect_exception(tarfile.LinkOutsideDestinationError) + else: + e = self.expect_exception( + tarfile.LinkFallbackError, + "link 'boom' would be extracted as a copy of " + + "'c/escape', which was rejected") + self.assertIsInstance(e.__cause__, + tarfile.LinkOutsideDestinationError) + for filter in 'tar', 'fully_trusted': + with self.subTest(filter), self.check_context(arc.open(), filter): + if not support.can_symlink(): + self.expect_file("a/t/dummy") + self.expect_file("b/") + self.expect_file("c/") + else: + self.expect_file("a/t/dummy") + self.expect_file("b/") + self.expect_file("a/t/escape", symlink_to='../../link_here') + self.expect_file("boom", symlink_to='../../link_here') + self.expect_file("c", symlink_to='b') + + def test_exfiltration_via_symlink(self): + # (CVE-2025-4138) + # Test changing symlinks that result in a symlink pointing outside + # the extraction directory, unless prevented by 'data' filter's + # normalization. + with ArchiveMaker() as arc: + arc.add("escape", symlink_to=os.path.join('link', 'link', '..', '..', 'link-here')) + arc.add("link", symlink_to='./') + + for filter in 'tar', 'data', 'fully_trusted': + with self.check_context(arc.open(), filter): + if support.can_symlink(): + self.expect_file("link", symlink_to='./') + if filter == 'data': + self.expect_file("escape", symlink_to='link-here') + else: + self.expect_file("escape", + symlink_to='link/link/../../link-here') + else: + # Nothing is extracted. + pass + + def test_chmod_outside_dir(self): + # (CVE-2024-12718) + # Test that members used for delayed updates of directory metadata + # are (re-)filtered. + with ArchiveMaker() as arc: + # "pwn" is a veeeery innocent symlink: + arc.add("a/pwn", symlink_to='.') + # But now "pwn" is also a directory, so it's scheduled to have its + # metadata updated later: + arc.add("a/pwn/", mode='drwxrwxrwx') + # Oops, "pwn" is not so innocent any more: + arc.add("a/pwn", symlink_to='x/../') + # Newly created symlink points to the dest dir, + # so it's OK for the "data" filter. + arc.add('a/x', symlink_to=('../')) + # But now "pwn" points outside the dest dir + + for filter in 'tar', 'data', 'fully_trusted': + with self.check_context(arc.open(), filter) as cc: + if not support.can_symlink(): + self.expect_file("a/pwn/") + elif filter == 'data': + self.expect_file("a/x", symlink_to='../') + self.expect_file("a/pwn", symlink_to='.') + else: + self.expect_file("a/x", symlink_to='../') + self.expect_file("a/pwn", symlink_to='x/../') + if sys.platform != "win32": + st_mode = cc.outerdir.stat().st_mode + self.assertNotEqual(st_mode & 0o777, 0o777) + + def test_link_fallback_normalizes(self): + # Make sure hardlink fallbacks work for non-normalized paths for all + # filters + with ArchiveMaker() as arc: + arc.add("dir/") + arc.add("dir/../afile") + arc.add("link1", hardlink_to='dir/../afile') + arc.add("link2", hardlink_to='dir/../dir/../afile') + + for filter in 'tar', 'data', 'fully_trusted': + with self.check_context(arc.open(), filter) as cc: + self.expect_file("dir/") + self.expect_file("afile") + self.expect_file("link1") + self.expect_file("link2") + def test_modes(self): # Test how file modes are extracted # (Note that the modes are ignored on platforms without working chmod) @@ -3566,7 +3844,7 @@ def test_tar_filter(self): # The 'tar' filter returns TarInfo objects with the same name/type. # (It can also fail for particularly "evil" input, but we don't have # that in the test archive.) - with tarfile.TarFile.open(tarname) as tar: + with tarfile.TarFile.open(tarname, encoding="iso8859-1") as tar: for tarinfo in tar.getmembers(): filtered = tarfile.tar_filter(tarinfo, '') self.assertIs(filtered.name, tarinfo.name) @@ -3575,7 +3853,7 @@ def test_tar_filter(self): def test_data_filter(self): # The 'data' filter either raises, or returns TarInfo with the same # name/type. - with tarfile.TarFile.open(tarname) as tar: + with tarfile.TarFile.open(tarname, encoding="iso8859-1") as tar: for tarinfo in tar.getmembers(): try: filtered = tarfile.data_filter(tarinfo, '') @@ -3704,13 +3982,13 @@ def valueerror_filter(tarinfo, path): # If errorlevel is 0, errors affected by errorlevel are ignored with self.check_context(arc.open(errorlevel=0), extracterror_filter): - self.expect_file('file') + pass with self.check_context(arc.open(errorlevel=0), filtererror_filter): - self.expect_file('file') + pass with self.check_context(arc.open(errorlevel=0), oserror_filter): - self.expect_file('file') + pass with self.check_context(arc.open(errorlevel=0), tarerror_filter): self.expect_exception(tarfile.TarError) @@ -3721,7 +3999,7 @@ def valueerror_filter(tarinfo, path): # If 1, all fatal errors are raised with self.check_context(arc.open(errorlevel=1), extracterror_filter): - self.expect_file('file') + pass with self.check_context(arc.open(errorlevel=1), filtererror_filter): self.expect_exception(tarfile.FilterError) diff --git a/Misc/NEWS.d/next/Security/2025-06-02-11-32-23.gh-issue-135034.RLGjbp.rst b/Misc/NEWS.d/next/Security/2025-06-02-11-32-23.gh-issue-135034.RLGjbp.rst new file mode 100644 index 000000000000000..e3f984afc215784 --- /dev/null +++ b/Misc/NEWS.d/next/Security/2025-06-02-11-32-23.gh-issue-135034.RLGjbp.rst @@ -0,0 +1,6 @@ +Fixes multiple issues that allowed ``tarfile`` extraction filters +(``filter="data"`` and ``filter="tar"``) to be bypassed using crafted +symlinks and hard links. + +Addresses CVE 2024-12718, CVE 2025-4138, CVE 2025-4330, and CVE 2025-4517. + From de950eac95ec5c7cb1cc6dc1380882baf84b7d21 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Wed, 25 Nov 2020 01:52:51 -0800 Subject: [PATCH 023/129] bpo-12800: tarfile: Restore fix from 011525ee9 (GH-21409) Restore fix from 011525ee92eb1c13ad1a62d28725a840e28f8160. (cherry picked from commit 4fedd7123eaf147edd55eabbbd72e0bcc8368e47) Co-authored-by: Julien Palard (cherry picked from commit 9d2c2a8e3b8fe18ee1568bfa4a419847b3e78575) --- Lib/tarfile.py | 3 +++ Lib/test/test_tarfile.py | 21 ++++++++----------- .../2020-07-09-11-32-28.bpo-12800.fNgWwx.rst | 4 ++++ 3 files changed, 16 insertions(+), 12 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2020-07-09-11-32-28.bpo-12800.fNgWwx.rst diff --git a/Lib/tarfile.py b/Lib/tarfile.py index ce6737e55286eb9..635e56b7957c292 100755 --- a/Lib/tarfile.py +++ b/Lib/tarfile.py @@ -2559,6 +2559,9 @@ def makelink_with_filter(self, tarinfo, targetpath, try: # For systems that support symbolic and hard links. if tarinfo.issym(): + if os.path.lexists(targetpath): + # Avoid FileExistsError on following os.symlink. + os.unlink(targetpath) os.symlink(tarinfo.linkname, targetpath) return else: diff --git a/Lib/test/test_tarfile.py b/Lib/test/test_tarfile.py index 8ff427e293f8c54..0d0e8ed418c71f2 100644 --- a/Lib/test/test_tarfile.py +++ b/Lib/test/test_tarfile.py @@ -1360,19 +1360,16 @@ def test_extractall_symlinks(self): with open(source_file,'w') as f: f.write('something\n') os.symlink(source_file, target_file) - tar = tarfile.open(temparchive,'w') - tar.add(source_file) - tar.add(target_file) - tar.close() + with tarfile.open(temparchive, 'w') as tar: + tar.add(source_file, arcname="source") + tar.add(target_file, arcname="symlink") # Let's extract it to the location which contains the symlink - tar = tarfile.open(temparchive,'r') - # this should not raise OSError: [Errno 17] File exists - try: - tar.extractall(path=tempdir) - except OSError: - self.fail("extractall failed with symlinked files") - finally: - tar.close() + with tarfile.open(temparchive, errorlevel=2) as tar: + # this should not raise OSError: [Errno 17] File exists + try: + tar.extractall(path=tempdir) + except OSError: + self.fail("extractall failed with symlinked files") finally: support.unlink(temparchive) support.rmtree(tempdir) diff --git a/Misc/NEWS.d/next/Library/2020-07-09-11-32-28.bpo-12800.fNgWwx.rst b/Misc/NEWS.d/next/Library/2020-07-09-11-32-28.bpo-12800.fNgWwx.rst new file mode 100644 index 000000000000000..fdd7c5e74f33a33 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2020-07-09-11-32-28.bpo-12800.fNgWwx.rst @@ -0,0 +1,4 @@ +Extracting a symlink from a tarball should succeed and overwrite the symlink +if it already exists. The fix is to remove the existing file or symlink +before extraction. Based on patch by Chris AtLee, Jeffrey Kintscher, and +Senthil Kumaran. From 0c64d04f42ecef6418d3e5773a8637d0d18f2b81 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Fri, 4 Jul 2025 00:06:00 +0300 Subject: [PATCH 024/129] [3.9] gh-135462: Fix quadratic complexity in processing special input in HTMLParser (GH-135464) (GH-135486) End-of-file errors are now handled according to the HTML5 specs -- comments and declarations are automatically closed, tags are ignored. (cherry picked from commit 6eb6c5dbfb528bd07d77b60fd71fd05d81d45c41) (cherry picked from commit 8d1b3dfa09135affbbf27fb8babcf3c11415df49) 3.7.17.x: 3.7 keeps the deprecated HTMLParser.unescape() method and its test. --- Lib/html/parser.py | 41 +++++--- Lib/test/test_htmlparser.py | 95 ++++++++++++++++--- ...-06-13-15-55-22.gh-issue-135462.KBeJpc.rst | 4 + 3 files changed, 117 insertions(+), 23 deletions(-) create mode 100644 Misc/NEWS.d/next/Security/2025-06-13-15-55-22.gh-issue-135462.KBeJpc.rst diff --git a/Lib/html/parser.py b/Lib/html/parser.py index de81879a631ac7e..e035a4a01364648 100644 --- a/Lib/html/parser.py +++ b/Lib/html/parser.py @@ -26,6 +26,7 @@ charref = re.compile('&#(?:[0-9]+|[xX][0-9a-fA-F]+)[^0-9a-fA-F]') starttagopen = re.compile('<[a-zA-Z]') +endtagopen = re.compile('') commentclose = re.compile(r'--\s*>') # Note: @@ -177,7 +178,7 @@ def goahead(self, end): k = self.parse_pi(i) elif startswith("', i + 1) - if k < 0: - k = rawdata.find('<', i + 1) - if k < 0: - k = i + 1 - else: - k += 1 - if self.convert_charrefs and not self.cdata_elem: - self.handle_data(unescape(rawdata[i:k])) + if starttagopen.match(rawdata, i): # < + letter + pass + elif startswith("'), - ('comment', '/img'), - ('endtag', 'html<')]) + ('data', '\n')]) def test_starttag_junk_chars(self): + self._run_check("<", [('data', '<')]) + self._run_check("<>", [('data', '<>')]) + self._run_check("< >", [('data', '< >')]) + self._run_check("< ", [('data', '< ')]) self._run_check("", []) + self._run_check("<$>", [('data', '<$>')]) self._run_check("", [('comment', '$')]) self._run_check("", [('endtag', 'a')]) + self._run_check("", [('starttag', 'a", [('endtag', 'a'", [('data', "'", []) + self._run_check("", [('starttag', 'a$b', [])]) self._run_check("", [('startendtag', 'a$b', [])]) self._run_check("", [('starttag', 'a$b', [])]) self._run_check("", [('startendtag', 'a$b', [])]) + self._run_check("", [('endtag', 'a$b')]) def test_slashes_in_starttag(self): self._run_check('', [('startendtag', 'a', [('foo', 'var')])]) @@ -580,13 +588,56 @@ def test_unescape_method(self): s = '""""""&#bad;' self.assertEqual(p.unescape(s), unescape(s)) - def test_broken_comments(self): - html = ('' + def test_eof_in_comments(self): + data = [ + ('', [('comment', '-!>')]), + ('' '' '' '') expected = [ + ('comment', 'ELEMENT br EMPTY'), ('comment', ' not really a comment '), ('comment', ' not a comment either --'), ('comment', ' -- close enough --'), @@ -641,6 +692,26 @@ def test_convert_charrefs_dropped_text(self): ('endtag', 'a'), ('data', ' bar & baz')] ) + @support.requires_resource('cpu') + def test_eof_no_quadratic_complexity(self): + # Each of these examples used to take about an hour. + # Now they take a fraction of a second. + def check(source): + parser = html.parser.HTMLParser() + parser.feed(source) + parser.close() + n = 120_000 + check(" Date: Sat, 12 Jul 2025 14:24:14 +0200 Subject: [PATCH 025/129] [3.9] gh-102555: Fix comment parsing in HTMLParser according to the HTML5 standard (GH-135664) (GH-136276) * "--!>" now ends the comment. * "-- >" no longer ends the comment. * Support abnormally ended empty comments "<-->" and "<--->". --------- (cherry picked from commit 8ac7613dc8b8f82253d7c0e2b6ef6ed703a0a1ee) Co-author: Kerim Kabirov Co-authored-by: Serhiy Storchaka Co-authored-by: Ezio Melotti (cherry picked from commit a63f2294b7c24097a8b9ea52477220f8eb070b0d) --- Lib/html/parser.py | 18 ++++++++++- Lib/test/test_htmlparser.py | 32 +++++++++++++++++-- ...-06-18-13-28-08.gh-issue-102555.nADrzJ.rst | 3 ++ 3 files changed, 50 insertions(+), 3 deletions(-) create mode 100644 Misc/NEWS.d/next/Security/2025-06-18-13-28-08.gh-issue-102555.nADrzJ.rst diff --git a/Lib/html/parser.py b/Lib/html/parser.py index e035a4a01364648..37210c012ff2f20 100644 --- a/Lib/html/parser.py +++ b/Lib/html/parser.py @@ -28,7 +28,8 @@ starttagopen = re.compile('<[a-zA-Z]') endtagopen = re.compile('') -commentclose = re.compile(r'--\s*>') +commentclose = re.compile(r'--!?>') +commentabruptclose = re.compile(r'-?>') # Note: # 1) if you change tagfind/attrfind remember to update locatestarttagend too; # 2) if you change tagfind/attrfind and/or locatestarttagend the parser will @@ -291,6 +292,21 @@ def parse_html_declaration(self, i): else: return self.parse_bogus_comment(i) + # Internal -- parse comment, return length or -1 if not terminated + # see https://html.spec.whatwg.org/multipage/parsing.html#comment-start-state + def parse_comment(self, i, report=True): + rawdata = self.rawdata + assert rawdata.startswith('" '' '' + '' '' + # abrupt-closing-of-empty-comment + '' + '' '' '' - '') + '' + '' + '' + '' + '' + '' + '' + # nested-comment + ' -->' + '' + '' + ) expected = [('comment', " I'm a valid comment "), ('comment', 'me too!'), ('comment', '--'), + ('comment', '-'), + ('comment', ''), + ('comment', ''), ('comment', ''), ('comment', '--I have many hyphens--'), ('comment', ' I have a > in the middle '), - ('comment', ' and I have -- in the middle! ')] + ('comment', ' and I have -- in the middle! '), + ('comment', 'incorrectly-closed-comment'), + ('comment', ''), + ('comment', '--!'), + ('comment', '-- >'), + ('comment', '-!>'), + ('comment', '!>'), + ('comment', ' '), + ('comment', '`` now ends the comment. ``-- >`` no longer ends the +comment. Support abnormally ended empty comments ``<-->`` and ``<--->``. From 84f0d1dc478b25cef0f95da86860845b20d5af10 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Sat, 12 Jul 2025 14:27:08 +0200 Subject: [PATCH 026/129] [3.9] gh-135661: Fix parsing start and end tags in HTMLParser according to the HTML5 standard (GH-135930) (GH-136268) (#136293) * Whitespaces no longer accepted between `` does not end the script section. * Vertical tabulation (`\v`) and non-ASCII whitespaces no longer recognized as whitespaces. The only whitespaces are `\t\n\r\f `. * Null character (U+0000) no longer ends the tag name. * Attributes and slashes after the tag name in end tags are now ignored, instead of terminating after the first `>` in quoted attribute value. E.g. ``. * Multiple slashes and whitespaces between the last attribute and closing `>` are now ignored in both start and end tags. E.g. ``. * Multiple `=` between attribute name and value are no longer collapsed. E.g. `` produces attribute "foo" with value "=bar". * Whitespaces between the `=` separator and attribute name or value are no longer ignored. E.g. `` produces two attributes "foo" and "=bar", both with value None; `` produces two attributes: "foo" with value "" and "bar" with value None. * Fix data loss after unclosed script or style tag (gh-86155). Also backport test.support.subTests() (gh-135120). --------- (cherry picked from commit 0243f97cbadec8d985e63b1daec5d1cbc850cae3) (cherry picked from commit c555f889c3558a0a8cd8d8ecc2b493014b88a700) Co-authored-by: Serhiy Storchaka Co-authored-by: Ezio Melotti Co-authored-by: Waylan Limberg (cherry picked from commit 6c972008610648dd46e19a8c3beb7638b977d870) 3.7.17.x: Lib/html/parser.py taken from 3.9 plus 3.7's deprecated unescape() method; test.support.subTests() is added without positional- only parameters (3.8+ syntax). --- Lib/html/parser.py | 147 +++++++++--------- Lib/test/support/__init__.py | 25 +++ Lib/test/test_htmlparser.py | 145 +++++++++++------ ...3-02-13-21-41-34.gh-issue-86155.ppIGSC.rst | 2 + ...-06-25-14-13-39.gh-issue-135661.idjQ0B.rst | 25 +++ 5 files changed, 223 insertions(+), 121 deletions(-) create mode 100644 Misc/NEWS.d/next/Security/2023-02-13-21-41-34.gh-issue-86155.ppIGSC.rst create mode 100644 Misc/NEWS.d/next/Security/2025-06-25-14-13-39.gh-issue-135661.idjQ0B.rst diff --git a/Lib/html/parser.py b/Lib/html/parser.py index 37210c012ff2f20..a4a269d7edfc51e 100644 --- a/Lib/html/parser.py +++ b/Lib/html/parser.py @@ -31,15 +31,43 @@ commentclose = re.compile(r'--!?>') commentabruptclose = re.compile(r'-?>') # Note: -# 1) if you change tagfind/attrfind remember to update locatestarttagend too; -# 2) if you change tagfind/attrfind and/or locatestarttagend the parser will +# 1) if you change tagfind/attrfind remember to update locatetagend too; +# 2) if you change tagfind/attrfind and/or locatetagend the parser will # explode, so don't do it. -# see http://www.w3.org/TR/html5/tokenization.html#tag-open-state -# and http://www.w3.org/TR/html5/tokenization.html#tag-name-state -tagfind_tolerant = re.compile(r'([a-zA-Z][^\t\n\r\f />\x00]*)(?:\s|/(?!>))*') -attrfind_tolerant = re.compile( - r'((?<=[\'"\s/])[^\s/>][^\s/=>]*)(\s*=+\s*' - r'(\'[^\']*\'|"[^"]*"|(?![\'"])[^>\s]*))?(?:\s|/(?!>))*') +# see the HTML5 specs section "13.2.5.6 Tag open state", +# "13.2.5.8 Tag name state" and "13.2.5.33 Attribute name state". +# https://html.spec.whatwg.org/multipage/parsing.html#tag-open-state +# https://html.spec.whatwg.org/multipage/parsing.html#tag-name-state +# https://html.spec.whatwg.org/multipage/parsing.html#attribute-name-state +tagfind_tolerant = re.compile(r'([a-zA-Z][^\t\n\r\f />]*)(?:[\t\n\r\f ]|/(?!>))*') +attrfind_tolerant = re.compile(r""" + ( + (?<=['"\t\n\r\f /])[^\t\n\r\f />][^\t\n\r\f /=>]* # attribute name + ) + (= # value indicator + ('[^']*' # LITA-enclosed value + |"[^"]*" # LIT-enclosed value + |(?!['"])[^>\t\n\r\f ]* # bare value + ) + )? + (?:[\t\n\r\f ]|/(?!>))* # possibly followed by a space +""", re.VERBOSE) +locatetagend = re.compile(r""" + [a-zA-Z][^\t\n\r\f />]* # tag name + [\t\n\r\f /]* # optional whitespace before attribute name + (?:(?<=['"\t\n\r\f /])[^\t\n\r\f />][^\t\n\r\f /=>]* # attribute name + (?:= # value indicator + (?:'[^']*' # LITA-enclosed value + |"[^"]*" # LIT-enclosed value + |(?!['"])[^>\t\n\r\f ]* # bare value + ) + )? + [\t\n\r\f /]* # possibly followed by a space + )* + >? +""", re.VERBOSE) +# The following variables are not used, but are temporarily left for +# backward compatibility. locatestarttagend_tolerant = re.compile(r""" <[a-zA-Z][^\t\n\r\f />\x00]* # tag name (?:[\s/]* # optional whitespace before attribute name @@ -49,15 +77,13 @@ |"[^"]*" # LIT-enclosed value |(?!['"])[^>\s]* # bare value ) - (?:\s*,)* # possibly followed by a comma + \s* # possibly followed by a space )?(?:\s|/(?!>))* )* )? \s* # trailing whitespace """, re.VERBOSE) endendtag = re.compile('>') -# the HTML 5 spec, section 8.1.2.2, doesn't allow spaces between -# ') @@ -124,7 +150,8 @@ def get_starttag_text(self): def set_cdata_mode(self, elem): self.cdata_elem = elem.lower() - self.interesting = re.compile(r'' % self.cdata_elem, re.I) + self.interesting = re.compile(r'])' % self.cdata_elem, + re.IGNORECASE|re.ASCII) def clear_cdata_mode(self): self.interesting = interesting_normal @@ -149,7 +176,7 @@ def goahead(self, end): # & near the end and see if it's followed by a space or ;. amppos = rawdata.rfind('&', max(i, n-34)) if (amppos >= 0 and - not re.compile(r'[\s;]').search(rawdata, amppos)): + not re.compile(r'[\t\n\r\f ;]').search(rawdata, amppos)): break # wait till we get all the text j = n else: @@ -262,7 +289,7 @@ def goahead(self, end): else: assert 0, "interesting.search() lied" # end while - if end and i < n and not self.cdata_elem: + if end and i < n: if self.convert_charrefs and not self.cdata_elem: self.handle_data(unescape(rawdata[i:n])) else: @@ -308,7 +335,7 @@ def parse_comment(self, i, report=True): return match.end() # Internal -- parse bogus comment, return length or -1 if not terminated - # see http://www.w3.org/TR/html5/tokenization.html#bogus-comment-state + # see https://html.spec.whatwg.org/multipage/parsing.html#bogus-comment-state def parse_bogus_comment(self, i, report=1): rawdata = self.rawdata assert rawdata[i:i+2] in ('": - return j + 1 - if next == "/": - if rawdata.startswith("/>", j): - return j + 2 - if rawdata.startswith("/", j): - # buffer boundary - return -1 - # else bogus input - if j > i: - return j - else: - return i + 1 - if next == "": - # end of input - return -1 - if next in ("abcdefghijklmnopqrstuvwxyz=/" - "ABCDEFGHIJKLMNOPQRSTUVWXYZ"): - # end of input in or before attribute value, or we have the - # '/' from a '/>' ending - return -1 - if j > i: - return j - else: - return i + 1 - raise AssertionError("we should not get here!") + match = locatetagend.match(rawdata, i+1) + assert match + j = match.end() + if rawdata[j-1] != ">": + return -1 + return j # Internal -- parse endtag, return end or -1 if incomplete def parse_endtag(self, i): + # See the HTML5 specs section "13.2.5.7 End tag open state" + # https://html.spec.whatwg.org/multipage/parsing.html#end-tag-open-state rawdata = self.rawdata assert rawdata[i:i+2] == " - if not match: + if rawdata.find('>', i+2) < 0: # fast check return -1 - gtpos = match.end() - match = endtagfind.match(rawdata, i) # - if not match: - if self.cdata_elem is not None: - self.handle_data(rawdata[i:gtpos]) - return gtpos - # find the name: w3.org/TR/html5/tokenization.html#tag-name-state - namematch = tagfind_tolerant.match(rawdata, i+2) - if not namematch: - # w3.org/TR/html5/tokenization.html#end-tag-open-state - if rawdata[i:i+3] == '': - return i+3 - else: - return self.parse_bogus_comment(i) - tagname = namematch.group(1).lower() - # consume and ignore other stuff between the name and the > - # Note: this is not 100% correct, since we might have things like - # , but looking for > after tha name should cover - # most of the cases and is much simpler - gtpos = rawdata.find('>', namematch.end()) - self.handle_endtag(tagname) - return gtpos+1 + if not endtagopen.match(rawdata, i): # ': # is ignored + # "missing-end-tag-name" parser error + return i+3 + else: + return self.parse_bogus_comment(i) - elem = match.group(1).lower() # script or style - if self.cdata_elem is not None: - if elem != self.cdata_elem: - self.handle_data(rawdata[i:gtpos]) - return gtpos + match = locatetagend.match(rawdata, i+2) + assert match + j = match.end() + if rawdata[j-1] != ">": + return -1 - self.handle_endtag(elem) + # find the name: "13.2.5.8 Tag name state" + # https://html.spec.whatwg.org/multipage/parsing.html#tag-name-state + match = tagfind_tolerant.match(rawdata, i+2) + assert match + tag = match.group(1).lower() + self.handle_endtag(tag) self.clear_cdata_mode() - return gtpos + return j # Overridable -- finish processing of start+end tag: def handle_startendtag(self, tag, attrs): diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index 646001ba27de8be..6f4306516a3a26a 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -1639,6 +1639,31 @@ def check_sizeof(test, o, size): % (type(o), result, size) test.assertEqual(result, size, msg) +def subTests(arg_names, arg_values, *, _do_cleanups=False): + """Run multiple subtests with different parameters. + """ + single_param = False + if isinstance(arg_names, str): + arg_names = arg_names.replace(',',' ').split() + if len(arg_names) == 1: + single_param = True + arg_values = tuple(arg_values) + def decorator(func): + if isinstance(func, type): + raise TypeError('subTests() can only decorate methods, not classes') + @functools.wraps(func) + def wrapper(self, *args, **kwargs): + for values in arg_values: + if single_param: + values = (values,) + subtest_kwargs = dict(zip(arg_names, values)) + with self.subTest(**subtest_kwargs): + func(self, *args, **kwargs, **subtest_kwargs) + if _do_cleanups: + self.doCleanups() + return wrapper + return decorator + #======================================================================= # Decorator for running a function in a different locale, correctly resetting # it afterwards. diff --git a/Lib/test/test_htmlparser.py b/Lib/test/test_htmlparser.py index 767ef0831c2c745..3df58acf9bfffcf 100644 --- a/Lib/test/test_htmlparser.py +++ b/Lib/test/test_htmlparser.py @@ -80,6 +80,13 @@ def handle_entityref(self, data): self.fail('This should never be called with convert_charrefs=True') +# The normal event collector normalizes the events in get_events, +# so we override it to return the original list of events. +class EventCollectorNoNormalize(EventCollector): + def get_events(self): + return self.events + + class TestCaseBase(unittest.TestCase): def get_collector(self): @@ -264,8 +271,7 @@ def test_get_starttag_text(self): ("starttag", "foo:bar", [("one", "1"), ("two", "2")]), ("starttag_text", s)]) - def test_cdata_content(self): - contents = [ + @support.subTests('content', [ ' ¬-an-entity-ref;', "", '

', @@ -278,44 +284,83 @@ def test_cdata_content(self): 'src="http://www.example.org/r=\'+new ' 'Date().getTime()+\'"><\\/s\'+\'cript>\');\n//]]>'), '\n\n', - 'foo = "";', '', - # these two should be invalid according to the HTML 5 spec, - # section 8.1.2.2 - #'foo = ', - #'foo = ', - ] - elements = ['script', 'style', 'SCRIPT', 'STYLE', 'Script', 'Style'] - for content in contents: - for element in elements: - element_lower = element.lower() - s = '<{element}>{content}'.format(element=element, - content=content) - self._run_check(s, [("starttag", element_lower, []), - ("data", content), - ("endtag", element_lower)]) - - def test_cdata_with_closing_tags(self): + 'foo = ""', + 'foo = ""', + 'foo = ""', + 'foo = ""', + 'foo = ""', + 'foo = ""', + ]) + def test_script_content(self, content): + s = f'' + self._run_check(s, [("starttag", "script", []), + ("data", content), + ("endtag", "script")]) + + @support.subTests('content', [ + 'a::before { content: ""; }', + 'a::before { content: "¬-an-entity-ref;"; }', + 'a::before { content: ""; }', + 'a::before { content: "\u2603"; }', + 'a::before { content: "< /style>"; }', + 'a::before { content: ""; }', + 'a::before { content: ""; }', + 'a::before { content: ""; }', + 'a::before { content: ""; }', + 'a::before { content: ""; }', + ]) + def test_style_content(self, content): + s = f'' + self._run_check(s, [("starttag", "style", []), + ("data", content), + ("endtag", "style")]) + + @support.subTests('endtag', ['script', 'SCRIPT', 'script ', 'script\n', + 'script/', 'script foo=bar', 'script foo=">"']) + def test_script_closing_tag(self, endtag): # see issue #13358 # make sure that HTMLParser calls handle_data only once for each CDATA. - # The normal event collector normalizes the events in get_events, - # so we override it to return the original list of events. - class Collector(EventCollector): - def get_events(self): - return self.events - content = """ ¬-an-entity-ref;

''""" - for element in [' script', 'script ', ' script ', - '\nscript', 'script\n', '\nscript\n']: - element_lower = element.lower().strip() - s = '{content}{tail}' + self._run_check(s, [("starttag", "script", []), + ("data", content if end else content + tail)], + collector=EventCollectorNoNormalize(convert_charrefs=False)) def test_comments(self): html = ("" @@ -433,7 +478,7 @@ def test_starttag_junk_chars(self): self._run_check("", [('comment', '$')]) self._run_check("", [('endtag', 'a')]) + self._run_check("", [('comment', ' a')]) self._run_check("", [('starttag', 'a", [('endtag', 'a', [('endtag', 'a')]) + self._run_check('', [('endtag', 'a')]) + def test_declaration_junk_chars(self): self._run_check("", [('decl', 'DOCTYPE foo $ ')]) @@ -551,15 +600,11 @@ def test_invalid_end_tags(self): self._run_check(html, expected) def test_broken_invalid_end_tag(self): - # This is technically wrong (the "> shouldn't be included in the 'data') - # but is probably not worth fixing it (in addition to all the cases of - # the previous test, it would require a full attribute parsing). - # see #13993 html = 'This confuses the parser' expected = [('starttag', 'b', []), ('data', 'This'), ('endtag', 'b'), - ('data', '"> confuses the parser')] + ('data', ' confuses the parser')] self._run_check(html, expected) def test_correct_detection_of_start_tags(self): @@ -586,7 +631,7 @@ def test_correct_detection_of_start_tags(self): html = '

The rain' expected = [ - ('starttag', 'div', [('style', ''), (',', None), ('foo', 'bar')]), + ('starttag', 'div', [('style', ''), (',', None), ('foo', None), ('=', None), ('"bar"', None)]), ('starttag', 'b', []), ('data', 'The '), ('starttag', 'a', [('href', 'some_url')]), @@ -748,9 +793,15 @@ def test_attr_syntax(self): ("starttag", "a", [("b", "v"), ("c", "v"), ("d", "v"), ("e", None)]) ] self._run_check("""""", output) - self._run_check("""""", output) - self._run_check("""""", output) - self._run_check("""""", output) + self._run_check("", [('starttag', 'a', [('foo', '=bar')])]) + self._run_check("", [('starttag', 'a', [('foo', None), ('=bar', None)])]) + self._run_check("", [('starttag', 'a', [('foo', None), ('=bar', None)])]) + self._run_check("", [('starttag', 'a', [('foo\v', 'bar')])]) + self._run_check("", [('starttag', 'a', [('foo\xa0', 'bar')])]) + self._run_check("", [('starttag', 'a', [('foo', ''), ('bar', None)])]) + self._run_check("", [('starttag', 'a', [('foo', ''), ('bar', None)])]) + self._run_check("", [('starttag', 'a', [('foo', '\vbar')])]) + self._run_check("", [('starttag', 'a', [('foo', '\xa0bar')])]) def test_attr_values(self): self._run_check("""""", @@ -759,6 +810,10 @@ def test_attr_values(self): ("d", "\txyz\n")])]) self._run_check("""""", [("starttag", "a", [("b", ""), ("c", "")])]) + self._run_check("", + [("starttag", "a", [("b", ""), ("c", "")])]) + self._run_check("", + [("starttag", "a", [("b", "\v"), ("c", "\xa0")])]) # Regression test for SF patch #669683. self._run_check("", [("starttag", "e", [("a", "rgb(1,2,3)")])]) @@ -830,7 +885,7 @@ def test_malformed_attributes(self): ('data', 'test - bad2'), ('endtag', 'a'), ('starttag', 'a', [('href', "test'\xa0style='color:red;bad3'")]), ('data', 'test - bad3'), ('endtag', 'a'), - ('starttag', 'a', [('href', "test'\xa0style='color:red;bad4'")]), + ('starttag', 'a', [('href', None), ('=', None), ("test' style", 'color:red;bad4')]), ('data', 'test - bad4'), ('endtag', 'a') ] self._run_check(html, expected) diff --git a/Misc/NEWS.d/next/Security/2023-02-13-21-41-34.gh-issue-86155.ppIGSC.rst b/Misc/NEWS.d/next/Security/2023-02-13-21-41-34.gh-issue-86155.ppIGSC.rst new file mode 100644 index 000000000000000..bb85481b2296970 --- /dev/null +++ b/Misc/NEWS.d/next/Security/2023-02-13-21-41-34.gh-issue-86155.ppIGSC.rst @@ -0,0 +1,2 @@ +:meth:`html.parser.HTMLParser.close` no longer loses data when the +````. + +* Multiple slashes and whitespaces between the last attribute and closing ``>`` + are now ignored in both start and end tags. E.g. ````. + +* Multiple ``=`` between attribute name and value are no longer collapsed. + E.g. ```` produces attribute "foo" with value "=bar". + +* Whitespaces between the ``=`` separator and attribute name or value are no + longer ignored. E.g. ```` produces two attributes "foo" and + "=bar", both with value None; ```` produces two attributes: + "foo" with value "" and "bar" with value None. From 56eebfb03a7c4fcee7d546ba627bd426305e499d Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Tue, 22 Jul 2025 11:58:28 +0200 Subject: [PATCH 027/129] [3.9] gh-135661: Fix parsing attributes with whitespaces around the "=" separator in HTMLParser (GH-136908) (GH-136922) This fixes a regression introduced in GH-135930. (cherry picked from commit dee650189497735edbc08a54edabb5b06ef1bd09) Co-authored-by: Serhiy Storchaka (cherry picked from commit 06fc882eac0e59220a7b8b127a1e7babe0055d45) --- Lib/html/parser.py | 4 +-- Lib/test/test_htmlparser.py | 28 +++++++++++-------- ...-06-25-14-13-39.gh-issue-135661.idjQ0B.rst | 5 ---- 3 files changed, 18 insertions(+), 19 deletions(-) diff --git a/Lib/html/parser.py b/Lib/html/parser.py index a4a269d7edfc51e..b8112a2fc49b7e9 100644 --- a/Lib/html/parser.py +++ b/Lib/html/parser.py @@ -44,7 +44,7 @@ ( (?<=['"\t\n\r\f /])[^\t\n\r\f />][^\t\n\r\f /=>]* # attribute name ) - (= # value indicator + ([\t\n\r\f ]*=[\t\n\r\f ]* # value indicator ('[^']*' # LITA-enclosed value |"[^"]*" # LIT-enclosed value |(?!['"])[^>\t\n\r\f ]* # bare value @@ -56,7 +56,7 @@ [a-zA-Z][^\t\n\r\f />]* # tag name [\t\n\r\f /]* # optional whitespace before attribute name (?:(?<=['"\t\n\r\f /])[^\t\n\r\f />][^\t\n\r\f /=>]* # attribute name - (?:= # value indicator + (?:[\t\n\r\f ]*=[\t\n\r\f ]* # value indicator (?:'[^']*' # LITA-enclosed value |"[^"]*" # LIT-enclosed value |(?!['"])[^>\t\n\r\f ]* # bare value diff --git a/Lib/test/test_htmlparser.py b/Lib/test/test_htmlparser.py index 3df58acf9bfffcf..965f07c7d8969ee 100644 --- a/Lib/test/test_htmlparser.py +++ b/Lib/test/test_htmlparser.py @@ -631,7 +631,7 @@ def test_correct_detection_of_start_tags(self): html = '
The rain' expected = [ - ('starttag', 'div', [('style', ''), (',', None), ('foo', None), ('=', None), ('"bar"', None)]), + ('starttag', 'div', [('style', ''), (',', None), ('foo', 'bar')]), ('starttag', 'b', []), ('data', 'The '), ('starttag', 'a', [('href', 'some_url')]), @@ -794,12 +794,12 @@ def test_attr_syntax(self): ] self._run_check("""""", output) self._run_check("", [('starttag', 'a', [('foo', '=bar')])]) - self._run_check("", [('starttag', 'a', [('foo', None), ('=bar', None)])]) - self._run_check("", [('starttag', 'a', [('foo', None), ('=bar', None)])]) + self._run_check("", [('starttag', 'a', [('foo', 'bar')])]) + self._run_check("", [('starttag', 'a', [('foo', 'bar')])]) self._run_check("", [('starttag', 'a', [('foo\v', 'bar')])]) self._run_check("", [('starttag', 'a', [('foo\xa0', 'bar')])]) - self._run_check("", [('starttag', 'a', [('foo', ''), ('bar', None)])]) - self._run_check("", [('starttag', 'a', [('foo', ''), ('bar', None)])]) + self._run_check("", [('starttag', 'a', [('foo', 'bar')])]) + self._run_check("", [('starttag', 'a', [('foo', 'bar')])]) self._run_check("", [('starttag', 'a', [('foo', '\vbar')])]) self._run_check("", [('starttag', 'a', [('foo', '\xa0bar')])]) @@ -810,8 +810,8 @@ def test_attr_values(self): ("d", "\txyz\n")])]) self._run_check("""""", [("starttag", "a", [("b", ""), ("c", "")])]) - self._run_check("", - [("starttag", "a", [("b", ""), ("c", "")])]) + self._run_check("", + [('starttag', 'a', [('b', 'x'), ('c', 'y')])]) self._run_check("", [("starttag", "a", [("b", "\v"), ("c", "\xa0")])]) # Regression test for SF patch #669683. @@ -880,13 +880,17 @@ def test_malformed_attributes(self): ) expected = [ ('starttag', 'a', [('href', "test'style='color:red;bad1'")]), - ('data', 'test - bad1'), ('endtag', 'a'), + ('data', 'test - bad1'), + ('endtag', 'a'), ('starttag', 'a', [('href', "test'+style='color:red;ba2'")]), - ('data', 'test - bad2'), ('endtag', 'a'), + ('data', 'test - bad2'), + ('endtag', 'a'), ('starttag', 'a', [('href', "test'\xa0style='color:red;bad3'")]), - ('data', 'test - bad3'), ('endtag', 'a'), - ('starttag', 'a', [('href', None), ('=', None), ("test' style", 'color:red;bad4')]), - ('data', 'test - bad4'), ('endtag', 'a') + ('data', 'test - bad3'), + ('endtag', 'a'), + ('starttag', 'a', [('href', "test'\xa0style='color:red;bad4'")]), + ('data', 'test - bad4'), + ('endtag', 'a'), ] self._run_check(html, expected) diff --git a/Misc/NEWS.d/next/Security/2025-06-25-14-13-39.gh-issue-135661.idjQ0B.rst b/Misc/NEWS.d/next/Security/2025-06-25-14-13-39.gh-issue-135661.idjQ0B.rst index b6f9e104e440472..27e886abdb58e56 100644 --- a/Misc/NEWS.d/next/Security/2025-06-25-14-13-39.gh-issue-135661.idjQ0B.rst +++ b/Misc/NEWS.d/next/Security/2025-06-25-14-13-39.gh-issue-135661.idjQ0B.rst @@ -18,8 +18,3 @@ according to the HTML5 standard. * Multiple ``=`` between attribute name and value are no longer collapsed. E.g. ```` produces attribute "foo" with value "=bar". - -* Whitespaces between the ``=`` separator and attribute name or value are no - longer ignored. E.g. ```` produces two attributes "foo" and - "=bar", both with value None; ```` produces two attributes: - "foo" with value "" and "bar" with value None. From 0e237d89160bd1094c577421b75719f5696097b9 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sat, 13 Sep 2025 22:34:15 +0200 Subject: [PATCH 028/129] [3.9] gh-130577: tarfile now validates archives to ensure member offsets are non-negative (GH-137027) (GH-137645) gh-130577: tarfile now validates archives to ensure member offsets are non-negative (GH-137027) (cherry picked from commit 7040aa54f14676938970e10c5f74ea93cd56aa38) Co-authored-by: Alexander Urieles Co-authored-by: Gregory P. Smith (cherry picked from commit 73f03e4808206f71eb6b92c579505a220942ef19) --- Lib/tarfile.py | 3 + Lib/test/test_tarfile.py | 156 ++++++++++++++++++ ...-07-23-00-35-29.gh-issue-130577.c7EITy.rst | 3 + 3 files changed, 162 insertions(+) create mode 100644 Misc/NEWS.d/next/Library/2025-07-23-00-35-29.gh-issue-130577.c7EITy.rst diff --git a/Lib/tarfile.py b/Lib/tarfile.py index 635e56b7957c292..9301aebcedb0b01 100755 --- a/Lib/tarfile.py +++ b/Lib/tarfile.py @@ -1585,6 +1585,9 @@ def _block(self, count): """Round up a byte count by BLOCKSIZE and return it, e.g. _block(834) => 1024. """ + # Only non-negative offsets are allowed + if count < 0: + raise InvalidHeaderError("invalid offset") blocks, remainder = divmod(count, BLOCKSIZE) if remainder: blocks += 1 diff --git a/Lib/test/test_tarfile.py b/Lib/test/test_tarfile.py index 0d0e8ed418c71f2..8bba3b67031b7ec 100644 --- a/Lib/test/test_tarfile.py +++ b/Lib/test/test_tarfile.py @@ -43,6 +43,7 @@ def md5sum(data): xzname = os.path.join(TEMPDIR, "testtar.tar.xz") tmpname = os.path.join(TEMPDIR, "tmp.tar") dotlessname = os.path.join(TEMPDIR, "testtar") +SPACE = b" " md5_regtype = "65f477c818ad9e15f7feab0c6d37742f" md5_sparse = "a54fbc4ca4f4399a90e1b27164012fc6" @@ -4033,6 +4034,161 @@ def valueerror_filter(tarinfo, path): self.expect_exception(TypeError) # errorlevel is not int +class OffsetValidationTests(unittest.TestCase): + tarname = tmpname + invalid_posix_header = ( + # name: 100 bytes + tarfile.NUL * tarfile.LENGTH_NAME + # mode, space, null terminator: 8 bytes + + b"000755" + SPACE + tarfile.NUL + # uid, space, null terminator: 8 bytes + + b"000001" + SPACE + tarfile.NUL + # gid, space, null terminator: 8 bytes + + b"000001" + SPACE + tarfile.NUL + # size, space: 12 bytes + + b"\xff" * 11 + SPACE + # mtime, space: 12 bytes + + tarfile.NUL * 11 + SPACE + # chksum: 8 bytes + + b"0011407" + tarfile.NUL + # type: 1 byte + + tarfile.REGTYPE + # linkname: 100 bytes + + tarfile.NUL * tarfile.LENGTH_LINK + # magic: 6 bytes, version: 2 bytes + + tarfile.POSIX_MAGIC + # uname: 32 bytes + + tarfile.NUL * 32 + # gname: 32 bytes + + tarfile.NUL * 32 + # devmajor, space, null terminator: 8 bytes + + tarfile.NUL * 6 + SPACE + tarfile.NUL + # devminor, space, null terminator: 8 bytes + + tarfile.NUL * 6 + SPACE + tarfile.NUL + # prefix: 155 bytes + + tarfile.NUL * tarfile.LENGTH_PREFIX + # padding: 12 bytes + + tarfile.NUL * 12 + ) + invalid_gnu_header = ( + # name: 100 bytes + tarfile.NUL * tarfile.LENGTH_NAME + # mode, null terminator: 8 bytes + + b"0000755" + tarfile.NUL + # uid, null terminator: 8 bytes + + b"0000001" + tarfile.NUL + # gid, space, null terminator: 8 bytes + + b"0000001" + tarfile.NUL + # size, space: 12 bytes + + b"\xff" * 11 + SPACE + # mtime, space: 12 bytes + + tarfile.NUL * 11 + SPACE + # chksum: 8 bytes + + b"0011327" + tarfile.NUL + # type: 1 byte + + tarfile.REGTYPE + # linkname: 100 bytes + + tarfile.NUL * tarfile.LENGTH_LINK + # magic: 8 bytes + + tarfile.GNU_MAGIC + # uname: 32 bytes + + tarfile.NUL * 32 + # gname: 32 bytes + + tarfile.NUL * 32 + # devmajor, null terminator: 8 bytes + + tarfile.NUL * 8 + # devminor, null terminator: 8 bytes + + tarfile.NUL * 8 + # padding: 167 bytes + + tarfile.NUL * 167 + ) + invalid_v7_header = ( + # name: 100 bytes + tarfile.NUL * tarfile.LENGTH_NAME + # mode, space, null terminator: 8 bytes + + b"000755" + SPACE + tarfile.NUL + # uid, space, null terminator: 8 bytes + + b"000001" + SPACE + tarfile.NUL + # gid, space, null terminator: 8 bytes + + b"000001" + SPACE + tarfile.NUL + # size, space: 12 bytes + + b"\xff" * 11 + SPACE + # mtime, space: 12 bytes + + tarfile.NUL * 11 + SPACE + # chksum: 8 bytes + + b"0010070" + tarfile.NUL + # type: 1 byte + + tarfile.REGTYPE + # linkname: 100 bytes + + tarfile.NUL * tarfile.LENGTH_LINK + # padding: 255 bytes + + tarfile.NUL * 255 + ) + valid_gnu_header = tarfile.TarInfo("filename").tobuf(tarfile.GNU_FORMAT) + data_block = b"\xff" * tarfile.BLOCKSIZE + + def _write_buffer(self, buffer): + with open(self.tarname, "wb") as f: + f.write(buffer) + + def _get_members(self, ignore_zeros=None): + with open(self.tarname, "rb") as f: + with tarfile.open( + mode="r", fileobj=f, ignore_zeros=ignore_zeros + ) as tar: + return tar.getmembers() + + def _assert_raises_read_error_exception(self): + with self.assertRaisesRegex( + tarfile.ReadError, "file could not be opened successfully" + ): + self._get_members() + + def test_invalid_offset_header_validations(self): + for tar_format, invalid_header in ( + ("posix", self.invalid_posix_header), + ("gnu", self.invalid_gnu_header), + ("v7", self.invalid_v7_header), + ): + with self.subTest(format=tar_format): + self._write_buffer(invalid_header) + self._assert_raises_read_error_exception() + + def test_early_stop_at_invalid_offset_header(self): + buffer = self.valid_gnu_header + self.invalid_gnu_header + self.valid_gnu_header + self._write_buffer(buffer) + members = self._get_members() + self.assertEqual(len(members), 1) + self.assertEqual(members[0].name, "filename") + self.assertEqual(members[0].offset, 0) + + def test_ignore_invalid_archive(self): + # 3 invalid headers with their respective data + buffer = (self.invalid_gnu_header + self.data_block) * 3 + self._write_buffer(buffer) + members = self._get_members(ignore_zeros=True) + self.assertEqual(len(members), 0) + + def test_ignore_invalid_offset_headers(self): + for first_block, second_block, expected_offset in ( + ( + (self.valid_gnu_header), + (self.invalid_gnu_header + self.data_block), + 0, + ), + ( + (self.invalid_gnu_header + self.data_block), + (self.valid_gnu_header), + 1024, + ), + ): + self._write_buffer(first_block + second_block) + members = self._get_members(ignore_zeros=True) + self.assertEqual(len(members), 1) + self.assertEqual(members[0].name, "filename") + self.assertEqual(members[0].offset, expected_offset) + + def setUpModule(): support.unlink(TEMPDIR) os.makedirs(TEMPDIR) diff --git a/Misc/NEWS.d/next/Library/2025-07-23-00-35-29.gh-issue-130577.c7EITy.rst b/Misc/NEWS.d/next/Library/2025-07-23-00-35-29.gh-issue-130577.c7EITy.rst new file mode 100644 index 000000000000000..342cabbc865dc46 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-07-23-00-35-29.gh-issue-130577.c7EITy.rst @@ -0,0 +1,3 @@ +:mod:`tarfile` now validates archives to ensure member offsets are +non-negative. (Contributed by Alexander Enrique Urieles Nieto in +:gh:`130577`.) From 3aa6279b13683e89b65e1c39e9d45b010fd5cae4 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sat, 13 Sep 2025 23:35:13 +0300 Subject: [PATCH 029/129] [3.9] gh-118350: Fix support of elements "textarea" and "title" in HTMLParser (GH-135310) (GH-137784) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit (cherry picked from commit 4d02f31cdd45d81b95540d9076222b709d4f2335) Co-authored-by: Timon Viola <44016238+timonviola@users.noreply.github.com> Co-authored-by: Łukasz Langa (cherry picked from commit 4dea0fb67b7064eb637d859de0b5d3ce4989bd1c) --- Lib/html/parser.py | 20 +++- Lib/test/test_htmlparser.py | 96 +++++++++++++++++++ ...-06-09-20-38-25.gh-issue-118350.KgWCcP.rst | 2 + 3 files changed, 113 insertions(+), 5 deletions(-) create mode 100644 Misc/NEWS.d/next/Security/2025-06-09-20-38-25.gh-issue-118350.KgWCcP.rst diff --git a/Lib/html/parser.py b/Lib/html/parser.py index b8112a2fc49b7e9..a4a270b0e383588 100644 --- a/Lib/html/parser.py +++ b/Lib/html/parser.py @@ -111,6 +111,7 @@ class HTMLParser(_markupbase.ParserBase): """ CDATA_CONTENT_ELEMENTS = ("script", "style") + RCDATA_CONTENT_ELEMENTS = ("textarea", "title") def __init__(self, *, convert_charrefs=True): """Initialize and reset this instance. @@ -127,6 +128,7 @@ def reset(self): self.lasttag = '???' self.interesting = interesting_normal self.cdata_elem = None + self._escapable = True _markupbase.ParserBase.reset(self) def feed(self, data): @@ -148,14 +150,20 @@ def get_starttag_text(self): """Return full source of start tag: '<...>'.""" return self.__starttag_text - def set_cdata_mode(self, elem): + def set_cdata_mode(self, elem, *, escapable=False): self.cdata_elem = elem.lower() - self.interesting = re.compile(r'])' % self.cdata_elem, - re.IGNORECASE|re.ASCII) + self._escapable = escapable + if escapable and not self.convert_charrefs: + self.interesting = re.compile(r'&|])' % self.cdata_elem, + re.IGNORECASE|re.ASCII) + else: + self.interesting = re.compile(r'])' % self.cdata_elem, + re.IGNORECASE|re.ASCII) def clear_cdata_mode(self): self.interesting = interesting_normal self.cdata_elem = None + self._escapable = True # Internal -- handle data as far as reasonable. May leave state # and data to be processed by a subsequent call. If 'end' is @@ -188,7 +196,7 @@ def goahead(self, end): break j = n if i < j: - if self.convert_charrefs and not self.cdata_elem: + if self.convert_charrefs and self._escapable: self.handle_data(unescape(rawdata[i:j])) else: self.handle_data(rawdata[i:j]) @@ -290,7 +298,7 @@ def goahead(self, end): assert 0, "interesting.search() lied" # end while if end and i < n: - if self.convert_charrefs and not self.cdata_elem: + if self.convert_charrefs and self._escapable: self.handle_data(unescape(rawdata[i:n])) else: self.handle_data(rawdata[i:n]) @@ -409,6 +417,8 @@ def parse_starttag(self, i): self.handle_starttag(tag, attrs) if tag in self.CDATA_CONTENT_ELEMENTS: self.set_cdata_mode(tag) + elif tag in self.RCDATA_CONTENT_ELEMENTS: + self.set_cdata_mode(tag, escapable=True) return endpos # Internal -- check to see if we have a complete starttag; return end diff --git a/Lib/test/test_htmlparser.py b/Lib/test/test_htmlparser.py index 965f07c7d8969ee..99d690716aa7486 100644 --- a/Lib/test/test_htmlparser.py +++ b/Lib/test/test_htmlparser.py @@ -316,6 +316,49 @@ def test_style_content(self, content): ("data", content), ("endtag", "style")]) + @support.subTests('content', [ + '', + "", + '', + '', + '', + '\u2603', + '< /title>', + '', + '', + '', + '', + '', + ]) + def test_title_content(self, content): + source = f"{content}" + self._run_check(source, [ + ("starttag", "title", []), + ("data", content), + ("endtag", "title"), + ]) + + @support.subTests('content', [ + '', + "", + '', + '', + '', + '\u2603', + '< /textarea>', + '', + '', + '', + '', + ]) + def test_textarea_content(self, content): + source = f"" + self._run_check(source, [ + ("starttag", "textarea", []), + ("data", content), + ("endtag", "textarea"), + ]) + @support.subTests('endtag', ['script', 'SCRIPT', 'script ', 'script\n', 'script/', 'script foo=bar', 'script foo=">"']) def test_script_closing_tag(self, endtag): @@ -345,6 +388,38 @@ def test_style_closing_tag(self, endtag): ("endtag", "style")], collector=EventCollectorNoNormalize(convert_charrefs=False)) + @support.subTests('endtag', ['title', 'TITLE', 'title ', 'title\n', + 'title/', 'title foo=bar', 'title foo=">"']) + def test_title_closing_tag(self, endtag): + content = "Egg & Spam" + s = f'{content}</{endtag}>' + self._run_check(s, [("starttag", "title", []), + ('data', '<!-- not a comment --><i>Egg & Spam</i>'), + ("endtag", "title")], + collector=EventCollectorNoNormalize(convert_charrefs=True)) + self._run_check(s, [("starttag", "title", []), + ('data', '<!-- not a comment --><i>Egg '), + ('entityref', 'amp'), + ('data', ' Spam</i>'), + ("endtag", "title")], + collector=EventCollectorNoNormalize(convert_charrefs=False)) + + @support.subTests('endtag', ['textarea', 'TEXTAREA', 'textarea ', 'textarea\n', + 'textarea/', 'textarea foo=bar', 'textarea foo=">"']) + def test_textarea_closing_tag(self, endtag): + content = "<!-- not a comment --><i>Egg & Spam</i>" + s = f'<TexTarEa>{content}</{endtag}>' + self._run_check(s, [("starttag", "textarea", []), + ('data', '<!-- not a comment --><i>Egg & Spam</i>'), + ("endtag", "textarea")], + collector=EventCollectorNoNormalize(convert_charrefs=True)) + self._run_check(s, [("starttag", "textarea", []), + ('data', '<!-- not a comment --><i>Egg '), + ('entityref', 'amp'), + ('data', ' Spam</i>'), + ("endtag", "textarea")], + collector=EventCollectorNoNormalize(convert_charrefs=False)) + @support.subTests('tail,end', [ ('', False), ('<', False), @@ -362,6 +437,27 @@ def test_eof_in_script(self, tail, end): ("data", content if end else content + tail)], collector=EventCollectorNoNormalize(convert_charrefs=False)) + @support.subTests('tail,end', [ + ('', False), + ('<', False), + ('</', False), + ('</t', False), + ('</title', False), + ('" '' diff --git a/Misc/NEWS.d/next/Security/2025-06-09-20-38-25.gh-issue-118350.KgWCcP.rst b/Misc/NEWS.d/next/Security/2025-06-09-20-38-25.gh-issue-118350.KgWCcP.rst new file mode 100644 index 000000000000000..6ad3caf33b22012 --- /dev/null +++ b/Misc/NEWS.d/next/Security/2025-06-09-20-38-25.gh-issue-118350.KgWCcP.rst @@ -0,0 +1,2 @@ +Fix support of escapable raw text mode (elements "textarea" and "title") +in :class:`html.parser.HTMLParser`. From 6f837c0a4318c494e11a62cd040620fee8289e1b Mon Sep 17 00:00:00 2001 From: Sebastian Pipping Date: Tue, 7 Oct 2025 13:52:10 +0200 Subject: [PATCH 030/129] [3.9] gh-139400: Make sure that parent parsers outlive their subparsers in `pyexpat` (GH-139403) (GH-139614) Within libexpat, a parser created via `XML_ExternalEntityParserCreate` is relying on its parent parser throughout its entire lifetime. Prior to this fix, is was possible for the parent parser to be garbage-collected too early. (cherry picked from commit 6edb2ddb5f3695cf4938979d645f31d7fba43ec8) (cherry picked from commit 598165e3739c68f7d08171b645b32c2201497aa0) 3.7.17.x: only the parser lifetime tests; the reparse deferral tests come with gh-115398. --- Lib/test/test_pyexpat.py | 36 +++++++++++++++++++ ...-09-29-00-01-28.gh-issue-139400.X2T-jO.rst | 4 +++ Modules/pyexpat.c | 26 ++++++++++++++ 3 files changed, 66 insertions(+) create mode 100644 Misc/NEWS.d/next/Security/2025-09-29-00-01-28.gh-issue-139400.X2T-jO.rst diff --git a/Lib/test/test_pyexpat.py b/Lib/test/test_pyexpat.py index 92fffc45f30a964..ce96f1bbfbc489b 100644 --- a/Lib/test/test_pyexpat.py +++ b/Lib/test/test_pyexpat.py @@ -729,5 +729,41 @@ def resolve_entity(context, base, system_id, public_id): self.assertEqual(handler_call_args, [("bar", "baz")]) +class ParentParserLifetimeTest(unittest.TestCase): + """ + Subparsers make use of their parent XML_Parser inside of Expat. + As a result, parent parsers need to outlive subparsers. + + See https://github.com/python/cpython/issues/139400. + """ + + def test_parent_parser_outlives_its_subparsers__single(self): + parser = expat.ParserCreate() + subparser = parser.ExternalEntityParserCreate(None) + + # Now try to cause garbage collection of the parent parser + # while it's still being referenced by a related subparser. + del parser + + def test_parent_parser_outlives_its_subparsers__multiple(self): + parser = expat.ParserCreate() + subparser_one = parser.ExternalEntityParserCreate(None) + subparser_two = parser.ExternalEntityParserCreate(None) + + # Now try to cause garbage collection of the parent parser + # while it's still being referenced by a related subparser. + del parser + + def test_parent_parser_outlives_its_subparsers__chain(self): + parser = expat.ParserCreate() + subparser = parser.ExternalEntityParserCreate(None) + subsubparser = subparser.ExternalEntityParserCreate(None) + + # Now try to cause garbage collection of the parent parsers + # while they are still being referenced by a related subparser. + del parser + del subparser + + if __name__ == "__main__": unittest.main() diff --git a/Misc/NEWS.d/next/Security/2025-09-29-00-01-28.gh-issue-139400.X2T-jO.rst b/Misc/NEWS.d/next/Security/2025-09-29-00-01-28.gh-issue-139400.X2T-jO.rst new file mode 100644 index 000000000000000..a5dea3b5f8147ab --- /dev/null +++ b/Misc/NEWS.d/next/Security/2025-09-29-00-01-28.gh-issue-139400.X2T-jO.rst @@ -0,0 +1,4 @@ +:mod:`xml.parsers.expat`: Make sure that parent Expat parsers are only +garbage-collected once they are no longer referenced by subparsers created +by :meth:`~xml.parsers.expat.xmlparser.ExternalEntityParserCreate`. +Patch by Sebastian Pipping. diff --git a/Modules/pyexpat.c b/Modules/pyexpat.c index ab3dac6db168404..8746e18e84e5691 100644 --- a/Modules/pyexpat.c +++ b/Modules/pyexpat.c @@ -56,6 +56,15 @@ typedef struct { PyObject_HEAD XML_Parser itself; + /* + * Strong reference to a parent `xmlparseobject` if this parser + * is a child parser. Set to NULL if this parser is a root parser. + * This is needed to keep the parent parser alive as long as it has + * at least one child parser. + * + * See https://github.com/python/cpython/issues/139400 for details. + */ + PyObject *parent; int ordered_attributes; /* Return attributes as a list. */ int specified_attributes; /* Report only specified attributes. */ int in_callback; /* Is a callback active? */ @@ -933,6 +942,12 @@ pyexpat_xmlparser_ExternalEntityParserCreate_impl(xmlparseobject *self, new_parser = PyObject_GC_New(xmlparseobject, &Xmlparsetype); if (new_parser == NULL) return NULL; + + // The new subparser will make use of the parent XML_Parser inside of Expat. + // So we need to take subparsers into account with the reference counting + // of their parent parser. + Py_INCREF(self); + new_parser->buffer_size = self->buffer_size; new_parser->buffer_used = 0; new_parser->buffer = NULL; @@ -942,6 +957,7 @@ pyexpat_xmlparser_ExternalEntityParserCreate_impl(xmlparseobject *self, new_parser->ns_prefixes = self->ns_prefixes; new_parser->itself = XML_ExternalEntityParserCreate(self->itself, context, encoding); + new_parser->parent = (PyObject *)self; new_parser->handlers = 0; new_parser->intern = self->intern; Py_XINCREF(new_parser->intern); @@ -951,11 +967,13 @@ pyexpat_xmlparser_ExternalEntityParserCreate_impl(xmlparseobject *self, new_parser->buffer = PyMem_Malloc(new_parser->buffer_size); if (new_parser->buffer == NULL) { Py_DECREF(new_parser); + Py_DECREF(self); return PyErr_NoMemory(); } } if (!new_parser->itself) { Py_DECREF(new_parser); + Py_DECREF(self); return PyErr_NoMemory(); } @@ -968,6 +986,7 @@ pyexpat_xmlparser_ExternalEntityParserCreate_impl(xmlparseobject *self, new_parser->handlers = PyMem_New(PyObject *, i); if (!new_parser->handlers) { Py_DECREF(new_parser); + Py_DECREF(self); return PyErr_NoMemory(); } clear_handlers(new_parser, 1); @@ -1186,6 +1205,7 @@ newxmlparseobject(const char *encoding, const char *namespace_separator, PyObjec /* namespace_separator is either NULL or contains one char + \0 */ self->itself = XML_ParserCreate_MM(encoding, &ExpatMemoryHandler, namespace_separator); + self->parent = NULL; if (self->itself == NULL) { PyErr_SetString(PyExc_RuntimeError, "XML_ParserCreate failed"); @@ -1223,6 +1243,7 @@ xmlparse_dealloc(xmlparseobject *self) if (self->itself != NULL) XML_ParserFree(self->itself); self->itself = NULL; + Py_CLEAR(self->parent); if (self->handlers != NULL) { for (i = 0; handler_info[i].name != NULL; i++) @@ -1493,6 +1514,7 @@ xmlparse_traverse(xmlparseobject *op, visitproc visit, void *arg) int i; for (i = 0; handler_info[i].name != NULL; i++) Py_VISIT(op->handlers[i]); + Py_VISIT(op->parent); return 0; } @@ -1501,6 +1523,10 @@ xmlparse_clear(xmlparseobject *op) { clear_handlers(op, 0); Py_CLEAR(op->intern); + // NOTE: We cannot call Py_CLEAR(op->parent) prior to calling + // XML_ParserFree(op->itself), or a subparser could lose its parent + // XML_Parser while still making use of it internally. + // https://github.com/python/cpython/issues/139400 return 0; } From bc5dc0917da2bfbbef4ef265cddd93b20750f383 Mon Sep 17 00:00:00 2001 From: Sebastian Pipping Date: Thu, 7 Mar 2024 00:03:30 +0100 Subject: [PATCH 031/129] [3.9] gh-115398: Expose Expat >=2.6.0 reparse deferral API (CVE-2023-52425) (GH-115623) (GH-116272) Allow controlling Expat >=2.6.0 reparse deferral (CVE-2023-52425) by adding five new methods: - `xml.etree.ElementTree.XMLParser.flush` - `xml.etree.ElementTree.XMLPullParser.flush` - `xml.parsers.expat.xmlparser.GetReparseDeferralEnabled` - `xml.parsers.expat.xmlparser.SetReparseDeferralEnabled` - `xml.sax.expatreader.ExpatParser.flush` Based on the "flush" idea from https://github.com/python/cpython/pull/115138#issuecomment-1932444270 . Includes code suggested-by: Snild Dolkow and by core dev Serhiy Storchaka. Co-authored-by: Gregory P. Smith (cherry picked from commit 200762426b4b9878c33cdef620aa1a51631dd5df) 3.7.17.x: pyexpat.c merged by hand; Modules/clinic/{pyexpat,_elementtree}.c.h regenerated with 3.7's Argument Clinic; test_xml_etree imports pyexpat. --- Doc/library/pyexpat.rst | 36 ++++++ Doc/library/xml.etree.elementtree.rst | 39 +++++++ Include/pyexpat.h | 4 +- Lib/test/test_pyexpat.py | 53 +++++++++ Lib/test/test_sax.py | 51 +++++++++ Lib/test/test_xml_etree.py | 108 ++++++++++++++---- Lib/xml/etree/ElementTree.py | 14 +++ Lib/xml/sax/expatreader.py | 14 +++ ...-02-18-03-14-40.gh-issue-115398.tzvxH8.rst | 8 ++ Modules/_elementtree.c | 33 ++++++ Modules/clinic/_elementtree.c.h | 19 ++- Modules/clinic/pyexpat.c.h | 48 +++++++- Modules/expat/pyexpatns.h | 1 + Modules/pyexpat.c | 53 +++++++++ 14 files changed, 455 insertions(+), 26 deletions(-) create mode 100644 Misc/NEWS.d/next/Security/2024-02-18-03-14-40.gh-issue-115398.tzvxH8.rst diff --git a/Doc/library/pyexpat.rst b/Doc/library/pyexpat.rst index e43b9aecd86835a..eb459385a71da7e 100644 --- a/Doc/library/pyexpat.rst +++ b/Doc/library/pyexpat.rst @@ -196,6 +196,42 @@ XMLParser Objects :exc:`ExpatError` to be raised with the :attr:`code` attribute set to ``errors.codes[errors.XML_ERROR_CANT_CHANGE_FEATURE_ONCE_PARSING]``. +.. method:: xmlparser.SetReparseDeferralEnabled(enabled) + + .. warning:: + + Calling ``SetReparseDeferralEnabled(False)`` has security implications, + as detailed below; please make sure to understand these consequences + prior to using the ``SetReparseDeferralEnabled`` method. + + Expat 2.6.0 introduced a security mechanism called "reparse deferral" + where instead of causing denial of service through quadratic runtime + from reparsing large tokens, reparsing of unfinished tokens is now delayed + by default until a sufficient amount of input is reached. + Due to this delay, registered handlers may — depending of the sizing of + input chunks pushed to Expat — no longer be called right after pushing new + input to the parser. Where immediate feedback and taking over responsiblity + of protecting against denial of service from large tokens are both wanted, + calling ``SetReparseDeferralEnabled(False)`` disables reparse deferral + for the current Expat parser instance, temporarily or altogether. + Calling ``SetReparseDeferralEnabled(True)`` allows re-enabling reparse + deferral. + + Note that :meth:`SetReparseDeferralEnabled` has been backported to some + prior releases of CPython as a security fix. Check for availability of + :meth:`SetReparseDeferralEnabled` using :func:`hasattr` if used in code + running across a variety of Python versions. + + .. versionadded:: 3.9.19 + +.. method:: xmlparser.GetReparseDeferralEnabled() + + Returns whether reparse deferral is currently enabled for the given + Expat parser instance. + + .. versionadded:: 3.9.19 + + :class:`xmlparser` objects have the following attributes: diff --git a/Doc/library/xml.etree.elementtree.rst b/Doc/library/xml.etree.elementtree.rst index ec38f09f76de091..f8eb7e051c147fb 100644 --- a/Doc/library/xml.etree.elementtree.rst +++ b/Doc/library/xml.etree.elementtree.rst @@ -163,6 +163,11 @@ data but would still like to have incremental parsing capabilities, take a look at :func:`iterparse`. It can be useful when you're reading a large XML document and don't want to hold it wholly in memory. +Where *immediate* feedback through events is wanted, calling method +:meth:`XMLPullParser.flush` can help reduce delay; +please make sure to study the related security notes. + + Finding interesting elements ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -1174,6 +1179,24 @@ XMLParser Objects Feeds data to the parser. *data* is encoded data. + + .. method:: flush() + + Triggers parsing of any previously fed unparsed data, which can be + used to ensure more immediate feedback, in particular with Expat >=2.6.0. + The implementation of :meth:`flush` temporarily disables reparse deferral + with Expat (if currently enabled) and triggers a reparse. + Disabling reparse deferral has security consequences; please see + :meth:`xml.parsers.expat.xmlparser.SetReparseDeferralEnabled` for details. + + Note that :meth:`flush` has been backported to some prior releases of + CPython as a security fix. Check for availability of :meth:`flush` + using :func:`hasattr` if used in code running across a variety of Python + versions. + + .. versionadded:: 3.9.19 + + :meth:`XMLParser.feed` calls *target*\'s ``start(tag, attrs_dict)`` method for each opening tag, its ``end(tag)`` method for each closing tag, and data is processed by method ``data(data)``. :meth:`XMLParser.close` calls @@ -1234,6 +1257,22 @@ XMLPullParser Objects Feed the given bytes data to the parser. + .. method:: flush() + + Triggers parsing of any previously fed unparsed data, which can be + used to ensure more immediate feedback, in particular with Expat >=2.6.0. + The implementation of :meth:`flush` temporarily disables reparse deferral + with Expat (if currently enabled) and triggers a reparse. + Disabling reparse deferral has security consequences; please see + :meth:`xml.parsers.expat.xmlparser.SetReparseDeferralEnabled` for details. + + Note that :meth:`flush` has been backported to some prior releases of + CPython as a security fix. Check for availability of :meth:`flush` + using :func:`hasattr` if used in code running across a variety of Python + versions. + + .. versionadded:: 3.9.19 + .. method:: close() Signal the parser that the data stream is terminated. Unlike diff --git a/Include/pyexpat.h b/Include/pyexpat.h index 07020b5dc964cb3..9824d099c3df7d0 100644 --- a/Include/pyexpat.h +++ b/Include/pyexpat.h @@ -48,8 +48,10 @@ struct PyExpat_CAPI enum XML_Status (*SetEncoding)(XML_Parser parser, const XML_Char *encoding); int (*DefaultUnknownEncodingHandler)( void *encodingHandlerData, const XML_Char *name, XML_Encoding *info); - /* might be none for expat < 2.1.0 */ + /* might be NULL for expat < 2.1.0 */ int (*SetHashSalt)(XML_Parser parser, unsigned long hash_salt); + /* might be NULL for expat < 2.6.0 */ + XML_Bool (*SetReparseDeferralEnabled)(XML_Parser parser, XML_Bool enabled); /* always add new stuff to the end! */ }; diff --git a/Lib/test/test_pyexpat.py b/Lib/test/test_pyexpat.py index ce96f1bbfbc489b..413c237679d99ff 100644 --- a/Lib/test/test_pyexpat.py +++ b/Lib/test/test_pyexpat.py @@ -764,6 +764,59 @@ def test_parent_parser_outlives_its_subparsers__chain(self): del parser del subparser +class ReparseDeferralTest(unittest.TestCase): + def test_getter_setter_round_trip(self): + parser = expat.ParserCreate() + enabled = (expat.version_info >= (2, 6, 0)) + + self.assertIs(parser.GetReparseDeferralEnabled(), enabled) + parser.SetReparseDeferralEnabled(False) + self.assertIs(parser.GetReparseDeferralEnabled(), False) + parser.SetReparseDeferralEnabled(True) + self.assertIs(parser.GetReparseDeferralEnabled(), enabled) + + def test_reparse_deferral_enabled(self): + if expat.version_info < (2, 6, 0): + self.skipTest(f'Expat {expat.version_info} does not ' + 'support reparse deferral') + + started = [] + + def start_element(name, _): + started.append(name) + + parser = expat.ParserCreate() + parser.StartElementHandler = start_element + self.assertTrue(parser.GetReparseDeferralEnabled()) + + for chunk in (b''): + parser.Parse(chunk, False) + + # The key test: Have handlers already fired? Expecting: no. + self.assertEqual(started, []) + + parser.Parse(b'', True) + + self.assertEqual(started, ['doc']) + + def test_reparse_deferral_disabled(self): + started = [] + + def start_element(name, _): + started.append(name) + + parser = expat.ParserCreate() + parser.StartElementHandler = start_element + if expat.version_info >= (2, 6, 0): + parser.SetReparseDeferralEnabled(False) + self.assertFalse(parser.GetReparseDeferralEnabled()) + + for chunk in (b''): + parser.Parse(chunk, False) + + # The key test: Have handlers already fired? Expecting: yes. + self.assertEqual(started, ['doc']) + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_sax.py b/Lib/test/test_sax.py index 3044960a0ed1654..fa1947b62cd884e 100644 --- a/Lib/test/test_sax.py +++ b/Lib/test/test_sax.py @@ -19,6 +19,7 @@ import codecs import gc import os.path +import pyexpat import shutil from urllib.error import URLError from test import support @@ -1168,6 +1169,56 @@ def test_expat_incremental_reset(self): self.assertEqual(result.getvalue(), start + b"text") + def test_flush_reparse_deferral_enabled(self): + if pyexpat.version_info < (2, 6, 0): + self.skipTest(f'Expat {pyexpat.version_info} does not support reparse deferral') + + result = BytesIO() + xmlgen = XMLGenerator(result) + parser = create_parser() + parser.setContentHandler(xmlgen) + + for chunk in (""): + parser.feed(chunk) + + self.assertEqual(result.getvalue(), start) # i.e. no elements started + self.assertTrue(parser._parser.GetReparseDeferralEnabled()) + + parser.flush() + + self.assertTrue(parser._parser.GetReparseDeferralEnabled()) + self.assertEqual(result.getvalue(), start + b"") + + parser.feed("") + parser.close() + + self.assertEqual(result.getvalue(), start + b"") + + def test_flush_reparse_deferral_disabled(self): + result = BytesIO() + xmlgen = XMLGenerator(result) + parser = create_parser() + parser.setContentHandler(xmlgen) + + for chunk in (""): + parser.feed(chunk) + + if pyexpat.version_info >= (2, 6, 0): + parser._parser.SetReparseDeferralEnabled(False) + + self.assertEqual(result.getvalue(), start) # i.e. no elements started + self.assertFalse(parser._parser.GetReparseDeferralEnabled()) + + parser.flush() + + self.assertFalse(parser._parser.GetReparseDeferralEnabled()) + self.assertEqual(result.getvalue(), start + b"") + + parser.feed("") + parser.close() + + self.assertEqual(result.getvalue(), start + b"") + # ===== Locator support def test_expat_locator_noinfo(self): diff --git a/Lib/test/test_xml_etree.py b/Lib/test/test_xml_etree.py index 5ba0de82cdf9a75..c251bda5bb074c3 100644 --- a/Lib/test/test_xml_etree.py +++ b/Lib/test/test_xml_etree.py @@ -11,6 +11,7 @@ import io import operator import pickle +import pyexpat import sys import types import unittest @@ -98,6 +99,7 @@ &entity; """ + def checkwarnings(*filters, quiet=False): def decorator(test): def newtest(*args, **kwargs): @@ -1047,40 +1049,49 @@ def test_html_empty_elems_serialization(self): class XMLPullParserTest(unittest.TestCase): - def _feed(self, parser, data, chunk_size=None): + def _feed(self, parser, data, chunk_size=None, flush=False): if chunk_size is None: parser.feed(data) else: for i in range(0, len(data), chunk_size): parser.feed(data[i:i+chunk_size]) + if flush: + parser.flush() def assert_event_tags(self, parser, expected): events = parser.read_events() self.assertEqual([(action, elem.tag) for action, elem in events], expected) - def test_simple_xml(self): - for chunk_size in (None, 1, 5): - with self.subTest(chunk_size=chunk_size): - parser = ET.XMLPullParser() - self.assert_event_tags(parser, []) - self._feed(parser, "\n", chunk_size) - self.assert_event_tags(parser, []) - self._feed(parser, - "\n text\n", chunk_size) - self.assert_event_tags(parser, [('end', 'element')]) - self._feed(parser, "texttail\n", chunk_size) - self._feed(parser, "\n", chunk_size) - self.assert_event_tags(parser, [ - ('end', 'element'), - ('end', 'empty-element'), - ]) - self._feed(parser, "\n", chunk_size) - self.assert_event_tags(parser, [('end', 'root')]) - self.assertIsNone(parser.close()) + def test_simple_xml(self, chunk_size=None, flush=False): + parser = ET.XMLPullParser() + self.assert_event_tags(parser, []) + self._feed(parser, "\n", chunk_size, flush) + self.assert_event_tags(parser, []) + self._feed(parser, + "\n text\n", chunk_size, flush) + self.assert_event_tags(parser, [('end', 'element')]) + self._feed(parser, "texttail\n", chunk_size, flush) + self._feed(parser, "\n", chunk_size, flush) + self.assert_event_tags(parser, [ + ('end', 'element'), + ('end', 'empty-element'), + ]) + self._feed(parser, "\n", chunk_size, flush) + self.assert_event_tags(parser, [('end', 'root')]) + self.assertIsNone(parser.close()) + + def test_simple_xml_chunk_1(self): + self.test_simple_xml(chunk_size=1, flush=True) + + def test_simple_xml_chunk_5(self): + self.test_simple_xml(chunk_size=5, flush=True) + + def test_simple_xml_chunk_22(self): + self.test_simple_xml(chunk_size=22) def test_feed_while_iterating(self): parser = ET.XMLPullParser() @@ -1198,6 +1209,57 @@ def test_unknown_event(self): with self.assertRaises(ValueError): ET.XMLPullParser(events=('start', 'end', 'bogus')) + def test_flush_reparse_deferral_enabled(self): + if pyexpat.version_info < (2, 6, 0): + self.skipTest(f'Expat {pyexpat.version_info} does not ' + 'support reparse deferral') + + parser = ET.XMLPullParser(events=('start', 'end')) + + for chunk in (""): + parser.feed(chunk) + + self.assert_event_tags(parser, []) # i.e. no elements started + if ET is pyET: + self.assertTrue(parser._parser._parser.GetReparseDeferralEnabled()) + + parser.flush() + + self.assert_event_tags(parser, [('start', 'doc')]) + if ET is pyET: + self.assertTrue(parser._parser._parser.GetReparseDeferralEnabled()) + + parser.feed("") + parser.close() + + self.assert_event_tags(parser, [('end', 'doc')]) + + def test_flush_reparse_deferral_disabled(self): + parser = ET.XMLPullParser(events=('start', 'end')) + + for chunk in (""): + parser.feed(chunk) + + if pyexpat.version_info >= (2, 6, 0): + if not ET is pyET: + self.skipTest(f'XMLParser.(Get|Set)ReparseDeferralEnabled ' + 'methods not available in C') + parser._parser._parser.SetReparseDeferralEnabled(False) + + self.assert_event_tags(parser, []) # i.e. no elements started + if ET is pyET: + self.assertFalse(parser._parser._parser.GetReparseDeferralEnabled()) + + parser.flush() + + self.assert_event_tags(parser, [('start', 'doc')]) + if ET is pyET: + self.assertFalse(parser._parser._parser.GetReparseDeferralEnabled()) + + parser.feed("") + parser.close() + + self.assert_event_tags(parser, [('end', 'doc')]) # # xinclude tests (samples from appendix C of the xinclude specification) diff --git a/Lib/xml/etree/ElementTree.py b/Lib/xml/etree/ElementTree.py index 87277045b525bb1..8d1ef84a2a42936 100644 --- a/Lib/xml/etree/ElementTree.py +++ b/Lib/xml/etree/ElementTree.py @@ -1298,6 +1298,11 @@ def read_events(self): else: yield event + def flush(self): + if self._parser is None: + raise ValueError("flush() called after end of stream") + self._parser.flush() + def XML(text, parser=None): """Parse XML document from string constant. @@ -1648,6 +1653,15 @@ def close(self): del self.parser, self._parser del self.target, self._target + def flush(self): + was_enabled = self.parser.GetReparseDeferralEnabled() + try: + self.parser.SetReparseDeferralEnabled(False) + self.parser.Parse(b"", False) + except self._error as v: + self._raiseerror(v) + finally: + self.parser.SetReparseDeferralEnabled(was_enabled) # Import the C accelerators try: diff --git a/Lib/xml/sax/expatreader.py b/Lib/xml/sax/expatreader.py index 5066ffc2fa51f02..e74b02acc68a052 100644 --- a/Lib/xml/sax/expatreader.py +++ b/Lib/xml/sax/expatreader.py @@ -220,6 +220,20 @@ def feed(self, data, isFinal = 0): # FIXME: when to invoke error()? self._err_handler.fatalError(exc) + def flush(self): + if self._parser is None: + return + + was_enabled = self._parser.GetReparseDeferralEnabled() + try: + self._parser.SetReparseDeferralEnabled(False) + self._parser.Parse(b"", False) + except expat.error as e: + exc = SAXParseException(expat.ErrorString(e.code), e, self) + self._err_handler.fatalError(exc) + finally: + self._parser.SetReparseDeferralEnabled(was_enabled) + def _close_source(self): source = self._source try: diff --git a/Misc/NEWS.d/next/Security/2024-02-18-03-14-40.gh-issue-115398.tzvxH8.rst b/Misc/NEWS.d/next/Security/2024-02-18-03-14-40.gh-issue-115398.tzvxH8.rst new file mode 100644 index 000000000000000..a40fcd35ef99ae4 --- /dev/null +++ b/Misc/NEWS.d/next/Security/2024-02-18-03-14-40.gh-issue-115398.tzvxH8.rst @@ -0,0 +1,8 @@ +Allow controlling Expat >=2.6.0 reparse deferral (CVE-2023-52425) by adding +five new methods: + +* :meth:`xml.etree.ElementTree.XMLParser.flush` +* :meth:`xml.etree.ElementTree.XMLPullParser.flush` +* :meth:`xml.parsers.expat.xmlparser.GetReparseDeferralEnabled` +* :meth:`xml.parsers.expat.xmlparser.SetReparseDeferralEnabled` +* :meth:`xml.sax.expatreader.ExpatParser.flush` diff --git a/Modules/_elementtree.c b/Modules/_elementtree.c index 9d5e635b9d75f42..104c9cf5fe8a2bb 100644 --- a/Modules/_elementtree.c +++ b/Modules/_elementtree.c @@ -3542,6 +3542,38 @@ _elementtree_XMLParser_close_impl(XMLParserObject *self) } } +/*[clinic input] +_elementtree.XMLParser.flush + +[clinic start generated code]*/ + +static PyObject * +_elementtree_XMLParser_flush_impl(XMLParserObject *self) +/*[clinic end generated code: output=42fdb8795ca24509 input=effbecdb28715949]*/ +{ + if (!_check_xmlparser(self)) { + return NULL; + } + + if (EXPAT(SetReparseDeferralEnabled) == NULL) { + Py_RETURN_NONE; + } + + // NOTE: The Expat parser in the C implementation of ElementTree is not + // exposed to the outside; as a result we known that reparse deferral + // is currently enabled, or we would not even have access to function + // XML_SetReparseDeferralEnabled in the first place (which we checked + // for, a few lines up). + + EXPAT(SetReparseDeferralEnabled)(self->parser, XML_FALSE); + + PyObject *res = expat_parse(self, "", 0, XML_FALSE); + + EXPAT(SetReparseDeferralEnabled)(self->parser, XML_TRUE); + + return res; +} + /*[clinic input] _elementtree.XMLParser.feed @@ -3988,6 +4020,7 @@ static PyTypeObject TreeBuilder_Type = { static PyMethodDef xmlparser_methods[] = { _ELEMENTTREE_XMLPARSER_FEED_METHODDEF _ELEMENTTREE_XMLPARSER_CLOSE_METHODDEF + _ELEMENTTREE_XMLPARSER_FLUSH_METHODDEF _ELEMENTTREE_XMLPARSER__PARSE_WHOLE_METHODDEF _ELEMENTTREE_XMLPARSER__SETEVENTS_METHODDEF _ELEMENTTREE_XMLPARSER_DOCTYPE_METHODDEF diff --git a/Modules/clinic/_elementtree.c.h b/Modules/clinic/_elementtree.c.h index 6b558876dd0bd89..c49f684334c59ad 100644 --- a/Modules/clinic/_elementtree.c.h +++ b/Modules/clinic/_elementtree.c.h @@ -672,6 +672,23 @@ _elementtree_XMLParser_close(XMLParserObject *self, PyObject *Py_UNUSED(ignored) return _elementtree_XMLParser_close_impl(self); } +PyDoc_STRVAR(_elementtree_XMLParser_flush__doc__, +"flush($self, /)\n" +"--\n" +"\n"); + +#define _ELEMENTTREE_XMLPARSER_FLUSH_METHODDEF \ + {"flush", (PyCFunction)_elementtree_XMLParser_flush, METH_NOARGS, _elementtree_XMLParser_flush__doc__}, + +static PyObject * +_elementtree_XMLParser_flush_impl(XMLParserObject *self); + +static PyObject * +_elementtree_XMLParser_flush(XMLParserObject *self, PyObject *Py_UNUSED(ignored)) +{ + return _elementtree_XMLParser_flush_impl(self); +} + PyDoc_STRVAR(_elementtree_XMLParser_feed__doc__, "feed($self, data, /)\n" "--\n" @@ -749,4 +766,4 @@ _elementtree_XMLParser__setevents(XMLParserObject *self, PyObject *const *args, exit: return return_value; } -/*[clinic end generated code: output=c5a85a88bbb5cc06 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=437833d15831087d input=a9049054013a1b77]*/ diff --git a/Modules/clinic/pyexpat.c.h b/Modules/clinic/pyexpat.c.h index 777b6f311458120..44196bde427f746 100644 --- a/Modules/clinic/pyexpat.c.h +++ b/Modules/clinic/pyexpat.c.h @@ -2,6 +2,52 @@ preserve [clinic start generated code]*/ +PyDoc_STRVAR(pyexpat_xmlparser_SetReparseDeferralEnabled__doc__, +"SetReparseDeferralEnabled($self, enabled, /)\n" +"--\n" +"\n" +"Enable/Disable reparse deferral; enabled by default with Expat >=2.6.0."); + +#define PYEXPAT_XMLPARSER_SETREPARSEDEFERRALENABLED_METHODDEF \ + {"SetReparseDeferralEnabled", (PyCFunction)pyexpat_xmlparser_SetReparseDeferralEnabled, METH_O, pyexpat_xmlparser_SetReparseDeferralEnabled__doc__}, + +static PyObject * +pyexpat_xmlparser_SetReparseDeferralEnabled_impl(xmlparseobject *self, + int enabled); + +static PyObject * +pyexpat_xmlparser_SetReparseDeferralEnabled(xmlparseobject *self, PyObject *arg) +{ + PyObject *return_value = NULL; + int enabled; + + if (!PyArg_Parse(arg, "p:SetReparseDeferralEnabled", &enabled)) { + goto exit; + } + return_value = pyexpat_xmlparser_SetReparseDeferralEnabled_impl(self, enabled); + +exit: + return return_value; +} + +PyDoc_STRVAR(pyexpat_xmlparser_GetReparseDeferralEnabled__doc__, +"GetReparseDeferralEnabled($self, /)\n" +"--\n" +"\n" +"Retrieve reparse deferral enabled status; always returns false with Expat <2.6.0."); + +#define PYEXPAT_XMLPARSER_GETREPARSEDEFERRALENABLED_METHODDEF \ + {"GetReparseDeferralEnabled", (PyCFunction)pyexpat_xmlparser_GetReparseDeferralEnabled, METH_NOARGS, pyexpat_xmlparser_GetReparseDeferralEnabled__doc__}, + +static PyObject * +pyexpat_xmlparser_GetReparseDeferralEnabled_impl(xmlparseobject *self); + +static PyObject * +pyexpat_xmlparser_GetReparseDeferralEnabled(xmlparseobject *self, PyObject *Py_UNUSED(ignored)) +{ + return pyexpat_xmlparser_GetReparseDeferralEnabled_impl(self); +} + PyDoc_STRVAR(pyexpat_xmlparser_Parse__doc__, "Parse($self, data, isfinal=False, /)\n" "--\n" @@ -289,4 +335,4 @@ pyexpat_ErrorString(PyObject *module, PyObject *arg) #ifndef PYEXPAT_XMLPARSER_USEFOREIGNDTD_METHODDEF #define PYEXPAT_XMLPARSER_USEFOREIGNDTD_METHODDEF #endif /* !defined(PYEXPAT_XMLPARSER_USEFOREIGNDTD_METHODDEF) */ -/*[clinic end generated code: output=34d02345deee104c input=a9049054013a1b77]*/ +/*[clinic end generated code: output=68f2282f9ca14a2f input=a9049054013a1b77]*/ diff --git a/Modules/expat/pyexpatns.h b/Modules/expat/pyexpatns.h index d45d9b6c4571595..8ee03ef07928157 100644 --- a/Modules/expat/pyexpatns.h +++ b/Modules/expat/pyexpatns.h @@ -108,6 +108,7 @@ #define XML_SetNotStandaloneHandler PyExpat_XML_SetNotStandaloneHandler #define XML_SetParamEntityParsing PyExpat_XML_SetParamEntityParsing #define XML_SetProcessingInstructionHandler PyExpat_XML_SetProcessingInstructionHandler +#define XML_SetReparseDeferralEnabled PyExpat_XML_SetReparseDeferralEnabled #define XML_SetReturnNSTriplet PyExpat_XML_SetReturnNSTriplet #define XML_SetSkippedEntityHandler PyExpat_XML_SetSkippedEntityHandler #define XML_SetStartCdataSectionHandler PyExpat_XML_SetStartCdataSectionHandler diff --git a/Modules/pyexpat.c b/Modules/pyexpat.c index 8746e18e84e5691..b8386758da5fd02 100644 --- a/Modules/pyexpat.c +++ b/Modules/pyexpat.c @@ -1,6 +1,7 @@ #include "Python.h" #include +#include #include "frameobject.h" #include "expat.h" @@ -73,6 +74,12 @@ typedef struct { /* NULL if not enabled */ int buffer_size; /* Size of buffer, in XML_Char units */ int buffer_used; /* Buffer units in use */ + bool reparse_deferral_enabled; /* Whether to defer reparsing of + unfinished XML tokens; a de-facto cache of + what Expat has the authority on, for lack + of a getter API function + "XML_GetReparseDeferralEnabled" in Expat + 2.6.0 */ PyObject *intern; /* Dictionary to intern strings */ PyObject **handlers; } xmlparseobject; @@ -714,6 +721,40 @@ get_parse_result(xmlparseobject *self, int rv) #define MAX_CHUNK_SIZE (1 << 20) +/*[clinic input] +pyexpat.xmlparser.SetReparseDeferralEnabled + + enabled: bool + / + +Enable/Disable reparse deferral; enabled by default with Expat >=2.6.0. +[clinic start generated code]*/ + +static PyObject * +pyexpat_xmlparser_SetReparseDeferralEnabled_impl(xmlparseobject *self, + int enabled) +/*[clinic end generated code: output=5ec539e3b63c8c49 input=021eb9e0bafc32c5]*/ +{ +#if XML_COMBINED_VERSION >= 20600 + XML_SetReparseDeferralEnabled(self->itself, enabled ? XML_TRUE : XML_FALSE); + self->reparse_deferral_enabled = (bool)enabled; +#endif + Py_RETURN_NONE; +} + +/*[clinic input] +pyexpat.xmlparser.GetReparseDeferralEnabled + +Retrieve reparse deferral enabled status; always returns false with Expat <2.6.0. +[clinic start generated code]*/ + +static PyObject * +pyexpat_xmlparser_GetReparseDeferralEnabled_impl(xmlparseobject *self) +/*[clinic end generated code: output=4e91312e88a595a8 input=54b5f11d32b20f3e]*/ +{ + return PyBool_FromLong(self->reparse_deferral_enabled); +} + /*[clinic input] pyexpat.xmlparser.Parse @@ -1118,6 +1159,8 @@ static struct PyMethodDef xmlparse_methods[] = { PYEXPAT_XMLPARSER_USEFOREIGNDTD_METHODDEF #endif PYEXPAT_XMLPARSER___DIR___METHODDEF + PYEXPAT_XMLPARSER_SETREPARSEDEFERRALENABLED_METHODDEF + PYEXPAT_XMLPARSER_GETREPARSEDEFERRALENABLED_METHODDEF {NULL, NULL} /* sentinel */ }; @@ -1200,6 +1243,11 @@ newxmlparseobject(const char *encoding, const char *namespace_separator, PyObjec self->handlers = NULL; self->intern = intern; Py_XINCREF(self->intern); +#if XML_COMBINED_VERSION >= 20600 + self->reparse_deferral_enabled = true; +#else + self->reparse_deferral_enabled = false; +#endif PyObject_GC_Track(self); /* namespace_separator is either NULL or contains one char + \0 */ @@ -1912,6 +1960,11 @@ MODULE_INITFUNC(void) #else capi.SetHashSalt = NULL; #endif +#if XML_COMBINED_VERSION >= 20600 + capi.SetReparseDeferralEnabled = XML_SetReparseDeferralEnabled; +#else + capi.SetReparseDeferralEnabled = NULL; +#endif /* export using capsule */ capi_object = PyCapsule_New(&capi, PyExpat_CAPSULE_NAME, NULL); From d1c766f43011200ea94bc9bcbd66cfaac2c5204e Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Wed, 27 Mar 2024 14:45:22 +0100 Subject: [PATCH 032/129] [3.9] gh-117187: Fix XML tests for vanilla Expat <2.6.0 (GH-117203) (GH-117247) This fixes XML unittest fallout from the https://github.com/python/cpython/issues/115398 security fix. When configured using `--with-system-expat` on systems with older pre 2.6.0 versions of libexpat, our unittests were failing. (cherry picked from commit 9f74e86c78853c101a23e938f8e32ea838d8f62e) Co-authored-by: Sebastian Pipping (cherry picked from commit f7c7e72a1c3bfe2f5c791a73557f9b29d369c89b) --- Lib/test/test_sax.py | 8 ++++---- Lib/test/test_xml_etree.py | 9 ++++----- .../Tests/2024-03-24-23-49-25.gh-issue-117187.eMLT5n.rst | 1 + 3 files changed, 9 insertions(+), 9 deletions(-) create mode 100644 Misc/NEWS.d/next/Tests/2024-03-24-23-49-25.gh-issue-117187.eMLT5n.rst diff --git a/Lib/test/test_sax.py b/Lib/test/test_sax.py index fa1947b62cd884e..13ebb4a2c62b9a8 100644 --- a/Lib/test/test_sax.py +++ b/Lib/test/test_sax.py @@ -1169,10 +1169,10 @@ def test_expat_incremental_reset(self): self.assertEqual(result.getvalue(), start + b"text") + @unittest.skipIf(pyexpat.version_info < (2, 6, 0), + f'Expat {pyexpat.version_info} does not ' + 'support reparse deferral') def test_flush_reparse_deferral_enabled(self): - if pyexpat.version_info < (2, 6, 0): - self.skipTest(f'Expat {pyexpat.version_info} does not support reparse deferral') - result = BytesIO() xmlgen = XMLGenerator(result) parser = create_parser() @@ -1205,8 +1205,8 @@ def test_flush_reparse_deferral_disabled(self): if pyexpat.version_info >= (2, 6, 0): parser._parser.SetReparseDeferralEnabled(False) + self.assertEqual(result.getvalue(), start) # i.e. no elements started - self.assertEqual(result.getvalue(), start) # i.e. no elements started self.assertFalse(parser._parser.GetReparseDeferralEnabled()) parser.flush() diff --git a/Lib/test/test_xml_etree.py b/Lib/test/test_xml_etree.py index c251bda5bb074c3..0cf38b13c24813b 100644 --- a/Lib/test/test_xml_etree.py +++ b/Lib/test/test_xml_etree.py @@ -1209,11 +1209,10 @@ def test_unknown_event(self): with self.assertRaises(ValueError): ET.XMLPullParser(events=('start', 'end', 'bogus')) + @unittest.skipIf(pyexpat.version_info < (2, 6, 0), + f'Expat {pyexpat.version_info} does not ' + 'support reparse deferral') def test_flush_reparse_deferral_enabled(self): - if pyexpat.version_info < (2, 6, 0): - self.skipTest(f'Expat {pyexpat.version_info} does not ' - 'support reparse deferral') - parser = ET.XMLPullParser(events=('start', 'end')) for chunk in (""): @@ -1245,8 +1244,8 @@ def test_flush_reparse_deferral_disabled(self): self.skipTest(f'XMLParser.(Get|Set)ReparseDeferralEnabled ' 'methods not available in C') parser._parser._parser.SetReparseDeferralEnabled(False) + self.assert_event_tags(parser, []) # i.e. no elements started - self.assert_event_tags(parser, []) # i.e. no elements started if ET is pyET: self.assertFalse(parser._parser._parser.GetReparseDeferralEnabled()) diff --git a/Misc/NEWS.d/next/Tests/2024-03-24-23-49-25.gh-issue-117187.eMLT5n.rst b/Misc/NEWS.d/next/Tests/2024-03-24-23-49-25.gh-issue-117187.eMLT5n.rst new file mode 100644 index 000000000000000..0c0b0e0f4433969 --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2024-03-24-23-49-25.gh-issue-117187.eMLT5n.rst @@ -0,0 +1 @@ +Fix XML tests for vanilla Expat <2.6.0. From 22e060349a813294023adb3def0132f4b386b81a Mon Sep 17 00:00:00 2001 From: Stan Ulbrych <89152624+StanFromIreland@users.noreply.github.com> Date: Tue, 7 Oct 2025 13:09:33 +0100 Subject: [PATCH 033/129] [3.9] gh-121227: Disallow setting an empty list for NPN (GH-137161) (cherry picked from commit a2cdbb6e8188ba9ba8b356b28d91bff60e86fe31) --- Lib/ssl.py | 2 ++ Lib/test/test_ssl.py | 6 ++++++ .../Security/2025-07-28-10-35-59.gh-issue-121227.Orp1wf.rst | 2 ++ 3 files changed, 10 insertions(+) create mode 100644 Misc/NEWS.d/next/Security/2025-07-28-10-35-59.gh-issue-121227.Orp1wf.rst diff --git a/Lib/ssl.py b/Lib/ssl.py index 4142b8d98b8ce4f..088162b9c23e2b7 100644 --- a/Lib/ssl.py +++ b/Lib/ssl.py @@ -435,6 +435,8 @@ def wrap_bio(self, incoming, outgoing, server_side=False, def set_npn_protocols(self, npn_protocols): protos = bytearray() + if not npn_protocols: + raise SSLError('NPN protocols must not be empty') for protocol in npn_protocols: b = bytes(protocol, 'ascii') if len(b) == 0 or len(b) > 255: diff --git a/Lib/test/test_ssl.py b/Lib/test/test_ssl.py index 920a11d96de487d..952ddae0946e937 100644 --- a/Lib/test/test_ssl.py +++ b/Lib/test/test_ssl.py @@ -4151,6 +4151,12 @@ def test_npn_protocols(self): if len(stats['server_npn_protocols']) else 'nothing' self.assertEqual(server_result, expected, msg % (server_result, "server")) + def test_empty_npn_protocols(self): + """npn_protocols cannot be empty, see CVE-2024-5642 & gh-121227""" + client_context, server_context, hostname = testing_context() + with self.assertRaises(ssl.SSLError): + server_context.set_npn_protocols([]) + def sni_contexts(self): server_context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) server_context.load_cert_chain(SIGNED_CERTFILE) diff --git a/Misc/NEWS.d/next/Security/2025-07-28-10-35-59.gh-issue-121227.Orp1wf.rst b/Misc/NEWS.d/next/Security/2025-07-28-10-35-59.gh-issue-121227.Orp1wf.rst new file mode 100644 index 000000000000000..6350f74a396f213 --- /dev/null +++ b/Misc/NEWS.d/next/Security/2025-07-28-10-35-59.gh-issue-121227.Orp1wf.rst @@ -0,0 +1,2 @@ +Raise an :exc:`SSL.SSLError` if an empty *protocols* argument is passed to +:meth:`ssl.SSLContext.set_npn_protocols` to fix ``CVE-2024-5642``. From e7e77782ed71caa5a3735fd6390215ea3a4c61e4 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Tue, 7 Oct 2025 21:15:04 +0200 Subject: [PATCH 034/129] [3.9] gh-135661: Fix CDATA section parsing in HTMLParser (GH-135665) (GH-137774) (GH-139661) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit "] ]>" and "]] >" no longer end the CDATA section. Make CDATA section parsing context depending. Add private method HTMLParser._set_support_cdata() to change the context. If called with True, "<[CDATA[" starts a CDATA section which ends with "]]>". If called with False, "<[CDATA[" starts a bogus comments which ends with ">". (cherry picked from commit 0cbbfc462119b9107b373c24d2bda5a1271bed36) (cherry picked from commit dcf24768c918c41821cda6fe6a1aa20ce26545dd) Co-authored-by: Serhiy Storchaka Co-authored-by: Łukasz Langa (cherry picked from commit ed904d5bbfa407771bf147c6eafd71f3b9a48869) 3.7.17.x: the moved attribute tests already exist in 3.7's test file; not duplicated. --- Lib/html/parser.py | 33 ++++++++- Lib/test/test_htmlparser.py | 74 +++++++++++++++++-- ...-06-18-13-34-55.gh-issue-135661.NZlpWf.rst | 5 ++ 3 files changed, 104 insertions(+), 8 deletions(-) create mode 100644 Misc/NEWS.d/next/Security/2025-06-18-13-34-55.gh-issue-135661.NZlpWf.rst diff --git a/Lib/html/parser.py b/Lib/html/parser.py index a4a270b0e383588..991b17bfdd02fde 100644 --- a/Lib/html/parser.py +++ b/Lib/html/parser.py @@ -128,6 +128,7 @@ def reset(self): self.lasttag = '???' self.interesting = interesting_normal self.cdata_elem = None + self._support_cdata = True self._escapable = True _markupbase.ParserBase.reset(self) @@ -165,6 +166,19 @@ def clear_cdata_mode(self): self.cdata_elem = None self._escapable = True + def _set_support_cdata(self, flag=True): + """Enable or disable support of the CDATA sections. + If enabled, "<[CDATA[" starts a CDATA section which ends with "]]>". + If disabled, "<[CDATA[" starts a bogus comments which ends with ">". + + This method is not called by default. Its purpose is to be called + in custom handle_starttag() and handle_endtag() methods, with + value that depends on the adjusted current node. + See https://html.spec.whatwg.org/multipage/parsing.html#markup-declaration-open-state + for details. + """ + self._support_cdata = flag + # Internal -- handle data as far as reasonable. May leave state # and data to be processed by a subsequent call. If 'end' is # true, force handling all data as if followed by EOF marker. @@ -239,7 +253,7 @@ def goahead(self, end): j -= len(suffix) break self.handle_comment(rawdata[i+4:j]) - elif startswith("', i+9) + if j < 0: + return -1 + self.unknown_decl(rawdata[i+3: j]) + return j + 3 elif rawdata[i:i+9].lower() == ' gtpos = rawdata.find('>', i+9) @@ -324,6 +342,15 @@ def parse_html_declaration(self, i): return -1 self.handle_decl(rawdata[i+2:gtpos]) return gtpos+1 + elif rawdata[i:i+3] == '', i+3) + if j < 0: + return -1 + if rawdata[j-1] == ']': + self.unknown_decl(rawdata[i+3: j-1]) + else: + self.handle_comment(rawdata[i+2: j]) + return j + 1 else: return self.parse_bogus_comment(i) diff --git a/Lib/test/test_htmlparser.py b/Lib/test/test_htmlparser.py index 99d690716aa7486..b95cdc1e3311d9b 100644 --- a/Lib/test/test_htmlparser.py +++ b/Lib/test/test_htmlparser.py @@ -9,10 +9,13 @@ class EventCollector(html.parser.HTMLParser): - def __init__(self, *args, **kw): + def __init__(self, *args, autocdata=False, **kw): + self.autocdata = autocdata self.events = [] self.append = self.events.append html.parser.HTMLParser.__init__(self, *args, **kw) + if autocdata: + self._set_support_cdata(False) def get_events(self): # Normalize the list of events so that buffer artefacts don't @@ -33,12 +36,16 @@ def get_events(self): def handle_starttag(self, tag, attrs): self.append(("starttag", tag, attrs)) + if self.autocdata and tag == 'svg': + self._set_support_cdata(True) def handle_startendtag(self, tag, attrs): self.append(("startendtag", tag, attrs)) def handle_endtag(self, tag): self.append(("endtag", tag)) + if self.autocdata and tag == 'svg': + self._set_support_cdata(False) # all other markup @@ -782,10 +789,6 @@ def test_eof_in_declarations(self): ('' '' @@ -847,8 +862,57 @@ def test_broken_condcoms(self): ('startendtag', 'img', [('src', 'mammoth.bmp')]), ('unknown decl', 'endif') ] + self._run_check(html, expected) + @support.subTests('content', [ + 'just some plain text', + '', + '¬-an-entity-ref;', + "", + '', + '[[I have many brackets]]', + 'I have a > in the middle', + 'I have a ]] in the middle', + '] ]>', + ']] >', + ('\n' + ' if (a < b && a > b) {\n' + ' printf("[How?]");\n' + ' }\n'), + ]) + def test_cdata_section_content(self, content): + # See "13.2.5.42 Markup declaration open state", + # "13.2.5.69 CDATA section state", and issue bpo-32876. + html = f'{content}' + expected = [ + ('starttag', 'svg', []), + ('starttag', 'text', [('y', '100')]), + ('unknown decl', 'CDATA[' + content), + ('endtag', 'text'), + ('endtag', 'svg'), + ] + self._run_check(html, expected) + self._run_check(html, expected, collector=EventCollector(autocdata=True)) + + def test_cdata_section(self): + # See "13.2.5.42 Markup declaration open state". + html = ('bar]]>' + 'foo<br>bar' + 'bar]]>') + expected = [ + ('comment', '[CDATA[foo'), + ('starttag', 'svg', []), + ('starttag', 'text', [('y', '100')]), + ('unknown decl', 'CDATA[foo
bar'), + ('endtag', 'text'), + ('endtag', 'svg'), + ('comment', '[CDATA[foo'), + ] + self._run_check(html, expected, collector=EventCollector(autocdata=True)) + def test_convert_charrefs_dropped_text(self): # #23144: make sure that all the events are triggered when # convert_charrefs is True, even if we don't call .close() diff --git a/Misc/NEWS.d/next/Security/2025-06-18-13-34-55.gh-issue-135661.NZlpWf.rst b/Misc/NEWS.d/next/Security/2025-06-18-13-34-55.gh-issue-135661.NZlpWf.rst new file mode 100644 index 000000000000000..fe000d936aae9d1 --- /dev/null +++ b/Misc/NEWS.d/next/Security/2025-06-18-13-34-55.gh-issue-135661.NZlpWf.rst @@ -0,0 +1,5 @@ +Fix CDATA section parsing in :class:`html.parser.HTMLParser` according to +the HTML5 standard: ``] ]>`` and ``]] >`` no longer end the CDATA section. +Add private method ``_set_support_cdata()`` which can be used to specify +how to parse ``<[CDATA[`` --- as a CDATA section in foreign content +(SVG or MathML) or as a bogus comment in the HTML namespace. From b5d3f7cdcc92db0bda527a133dd9d6ac8f10243f Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Tue, 7 Oct 2025 21:16:10 +0200 Subject: [PATCH 035/129] [3.9] gh-139700: Check consistency of the zip64 end of central directory record (GH-139702) (GH-139708) (#139715) Support records with "zip64 extensible data" if there are no bytes prepended to the ZIP file. (cherry picked from commit 333d4a6f4967d3ace91492a39ededbcf3faa76a6) (cherry picked from commit 162997bb70e067668c039700141770687bc8f267) Co-authored-by: Serhiy Storchaka (cherry picked from commit 76437ac248ad8ca44e9bf697b02b1e2241df2196) --- Lib/test/test_zipfile.py | 82 ++++++++++++++++++- Lib/zipfile.py | 51 +++++++----- ...-10-07-19-31-34.gh-issue-139700.vNHU1O.rst | 3 + 3 files changed, 113 insertions(+), 23 deletions(-) create mode 100644 Misc/NEWS.d/next/Security/2025-10-07-19-31-34.gh-issue-139700.vNHU1O.rst diff --git a/Lib/test/test_zipfile.py b/Lib/test/test_zipfile.py index 16bfb4a80907d18..357b20fcb36838f 100644 --- a/Lib/test/test_zipfile.py +++ b/Lib/test/test_zipfile.py @@ -806,6 +806,8 @@ def make_zip64_file( self, file_size_64_set=False, file_size_extra=False, compress_size_64_set=False, compress_size_extra=False, header_offset_64_set=False, header_offset_extra=False, + extensible_data=b'', + end_of_central_dir_size=None, offset_to_end_of_central_dir=None, ): """Generate bytes sequence for a zip with (incomplete) zip64 data. @@ -859,6 +861,12 @@ def make_zip64_file( central_dir_size = struct.pack(' 1: raise BadZipFile("zipfiles that span multiple disks are not supported") - # Assume no 'zip64 extensible data' - fpin.seek(offset - sizeEndCentDir64Locator - sizeEndCentDir64, 2) + offset -= sizeEndCentDir64 + if reloff > offset: + raise BadZipFile("Corrupt zip64 end of central directory locator") + # First, check the assumption that there is no prepended data. + fpin.seek(reloff) + extrasz = offset - reloff data = fpin.read(sizeEndCentDir64) if len(data) != sizeEndCentDir64: - return endrec + raise OSError("Unknown I/O error") + if not data.startswith(stringEndArchive64) and reloff != offset: + # Since we already have seen the Zip64 EOCD Locator, it's + # possible we got here because there is prepended data. + # Assume no 'zip64 extensible data' + fpin.seek(offset) + extrasz = 0 + data = fpin.read(sizeEndCentDir64) + if len(data) != sizeEndCentDir64: + raise OSError("Unknown I/O error") + if not data.startswith(stringEndArchive64): + raise BadZipFile("Zip64 end of central directory record not found") + sig, sz, create_version, read_version, disk_num, disk_dir, \ dircount, dircount2, dirsize, diroffset = \ struct.unpack(structEndArchive64, data) - if sig != stringEndArchive64: - return endrec + if (diroffset + dirsize != reloff or + sz + 12 != sizeEndCentDir64 + extrasz): + raise BadZipFile("Corrupt zip64 end of central directory record") # Update the original endrec using data from the ZIP64 record endrec[_ECD_SIGNATURE] = sig @@ -246,6 +262,7 @@ def _EndRecData64(fpin, offset, endrec): endrec[_ECD_ENTRIES_TOTAL] = dircount2 endrec[_ECD_SIZE] = dirsize endrec[_ECD_OFFSET] = diroffset + endrec[_ECD_LOCATION] = offset - extrasz return endrec @@ -279,7 +296,7 @@ def _EndRecData(fpin): endrec.append(filesize - sizeEndCentDir) # Try to read the "Zip64 end of central directory" structure - return _EndRecData64(fpin, -sizeEndCentDir, endrec) + return _EndRecData64(fpin, filesize - sizeEndCentDir, endrec) # Either this is not a ZIP file, or it is a ZIP file with an archive # comment. Search the end of the file for the "end of central directory" @@ -303,8 +320,7 @@ def _EndRecData(fpin): endrec.append(maxCommentStart + start) # Try to read the "Zip64 end of central directory" structure - return _EndRecData64(fpin, maxCommentStart + start - filesize, - endrec) + return _EndRecData64(fpin, maxCommentStart + start, endrec) # Unable to find a valid end of central directory structure return None @@ -1333,9 +1349,6 @@ def _RealGetContents(self): # "concat" is zero, unless zip was concatenated to another file concat = endrec[_ECD_LOCATION] - size_cd - offset_cd - if endrec[_ECD_SIGNATURE] == stringEndArchive64: - # If Zip64 extension structures are present, account for them - concat -= (sizeEndCentDir64 + sizeEndCentDir64Locator) if self.debug > 2: inferred = concat + offset_cd @@ -1925,7 +1938,7 @@ def _write_end_record(self): " would require ZIP64 extensions") zip64endrec = struct.pack( structEndArchive64, stringEndArchive64, - 44, 45, 45, 0, 0, centDirCount, centDirCount, + sizeEndCentDir64 - 12, 45, 45, 0, 0, centDirCount, centDirCount, centDirSize, centDirOffset) self.fp.write(zip64endrec) diff --git a/Misc/NEWS.d/next/Security/2025-10-07-19-31-34.gh-issue-139700.vNHU1O.rst b/Misc/NEWS.d/next/Security/2025-10-07-19-31-34.gh-issue-139700.vNHU1O.rst new file mode 100644 index 000000000000000..a8e7a1f1878c6bb --- /dev/null +++ b/Misc/NEWS.d/next/Security/2025-10-07-19-31-34.gh-issue-139700.vNHU1O.rst @@ -0,0 +1,3 @@ +Check consistency of the zip64 end of central directory record. Support +records with "zip64 extensible data" if there are no bytes prepended to the +ZIP file. From 221e4e643f36c6adc2065aaa226cafe356ca8ca6 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Fri, 31 Oct 2025 15:16:40 +0100 Subject: [PATCH 036/129] [3.9] gh-136063: fix quadratic-complexity parsing in `email.message._parseparam` (GH-136072) (GH-140832) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit (cherry picked from commit 680a5d070f59798bb88a1bb6eb027482b8d85c34) Co-authored-by: Bénédikt Tran <10796600+picnixz@users.noreply.github.com> (cherry picked from commit 90719e3c437c35c8d8aeace0817de90bc80c5967) --- Lib/email/message.py | 26 ++++++++++++------- Lib/test/test_email/test_email.py | 21 +++++++++++++++ ...-06-28-13-23-53.gh-issue-136063.aGk0Jv.rst | 2 ++ 3 files changed, 39 insertions(+), 10 deletions(-) create mode 100644 Misc/NEWS.d/next/Security/2025-06-28-13-23-53.gh-issue-136063.aGk0Jv.rst diff --git a/Lib/email/message.py b/Lib/email/message.py index 12626026179608e..89259e3c9b1cbae 100644 --- a/Lib/email/message.py +++ b/Lib/email/message.py @@ -74,19 +74,25 @@ def _parseparam(s): # RDM This might be a Header, so for now stringify it. s = ';' + str(s) plist = [] - while s[:1] == ';': - s = s[1:] - end = s.find(';') - while end > 0 and (s.count('"', 0, end) - s.count('\\"', 0, end)) % 2: - end = s.find(';', end + 1) + start = 0 + while s.find(';', start) == start: + start += 1 + end = s.find(';', start) + ind, diff = start, 0 + while end > 0: + diff += s.count('"', ind, end) - s.count('\\"', ind, end) + if diff % 2 == 0: + break + end, ind = ind, s.find(';', end + 1) if end < 0: end = len(s) - f = s[:end] - if '=' in f: - i = f.index('=') - f = f[:i].strip().lower() + '=' + f[i+1:].strip() + i = s.find('=', start, end) + if i == -1: + f = s[start:end] + else: + f = s[start:i].rstrip().lower() + '=' + s[i+1:end].lstrip() plist.append(f.strip()) - s = s[end:] + start = end return plist diff --git a/Lib/test/test_email/test_email.py b/Lib/test/test_email/test_email.py index 56a80631c97f188..f414f57f0587c54 100644 --- a/Lib/test/test_email/test_email.py +++ b/Lib/test/test_email/test_email.py @@ -410,6 +410,27 @@ def test_get_param_with_quotes(self): "Content-Type: foo; bar*0=\"baz\\\"foobar\"; bar*1=\"\\\"baz\"") self.assertEqual(msg.get_param('bar'), 'baz"foobar"baz') + def test_get_param_linear_complexity(self): + # Ensure that email.message._parseparam() is fast. + # See https://github.com/python/cpython/issues/136063. + N = 100_000 + for s, r in [ + ("", ""), + ("foo=bar", "foo=bar"), + (" FOO = bar ", "foo=bar"), + ]: + with self.subTest(s=s, r=r, N=N): + src = f'{s};' * (N - 1) + s + res = email.message._parseparam(src) + self.assertEqual(len(res), N) + self.assertEqual(len(set(res)), 1) + self.assertEqual(res[0], r) + + # This will be considered as a single parameter. + malformed = 's="' + ';' * (N - 1) + res = email.message._parseparam(malformed) + self.assertEqual(res, [malformed]) + def test_field_containment(self): msg = email.message_from_string('Header: exists') self.assertIn('header', msg) diff --git a/Misc/NEWS.d/next/Security/2025-06-28-13-23-53.gh-issue-136063.aGk0Jv.rst b/Misc/NEWS.d/next/Security/2025-06-28-13-23-53.gh-issue-136063.aGk0Jv.rst new file mode 100644 index 000000000000000..940a3ad5a72f68a --- /dev/null +++ b/Misc/NEWS.d/next/Security/2025-06-28-13-23-53.gh-issue-136063.aGk0Jv.rst @@ -0,0 +1,2 @@ +:mod:`email.message`: ensure linear complexity for legacy HTTP parameters +parsing. Patch by Bénédikt Tran. From 330e6c30cd8ef5721cd2eb4119ec967609850276 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Langa?= Date: Fri, 31 Oct 2025 17:05:53 +0100 Subject: [PATCH 037/129] [3.9] gh-136065: Fix quadratic complexity in os.path.expandvars() (GH-134952) (GH-140839) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit (cherry picked from commit f029e8db626ddc6e3a3beea4eff511a71aaceb5c) Co-authored-by: Serhiy Storchaka Co-authored-by: Łukasz Langa (cherry picked from commit 2e6150adccaaf5bd95d4c19dfd04a36e0b325d8c) 3.7.17.x: test imports adapted (no ALLOW_MISSING import needed). --- Lib/ntpath.py | 126 ++++++------------ Lib/posixpath.py | 43 +++--- Lib/test/test_genericpath.py | 19 ++- Lib/test/test_ntpath.py | 22 ++- ...-05-30-22-33-27.gh-issue-136065.bu337o.rst | 1 + 5 files changed, 96 insertions(+), 115 deletions(-) create mode 100644 Misc/NEWS.d/next/Security/2025-05-30-22-33-27.gh-issue-136065.bu337o.rst diff --git a/Lib/ntpath.py b/Lib/ntpath.py index bcea255e71d7f40..b40989bf53259e4 100644 --- a/Lib/ntpath.py +++ b/Lib/ntpath.py @@ -334,17 +334,23 @@ def expanduser(path): # XXX With COMMAND.COM you can use any characters in a variable name, # XXX except '^|<>='. +_varpattern = r"'[^']*'?|%(%|[^%]*%?)|\$(\$|[-\w]+|\{[^}]*\}?)" +_varsub = None +_varsubb = None + def expandvars(path): """Expand shell variables of the forms $var, ${var} and %var%. Unknown variables are left unchanged.""" path = os.fspath(path) + global _varsub, _varsubb if isinstance(path, bytes): if b'$' not in path and b'%' not in path: return path - import string - varchars = bytes(string.ascii_letters + string.digits + '_-', 'ascii') - quote = b'\'' + if not _varsubb: + import re + _varsubb = re.compile(_varpattern.encode(), re.ASCII).sub + sub = _varsubb percent = b'%' brace = b'{' rbrace = b'}' @@ -353,94 +359,44 @@ def expandvars(path): else: if '$' not in path and '%' not in path: return path - import string - varchars = string.ascii_letters + string.digits + '_-' - quote = '\'' + if not _varsub: + import re + _varsub = re.compile(_varpattern, re.ASCII).sub + sub = _varsub percent = '%' brace = '{' rbrace = '}' dollar = '$' environ = os.environ - res = path[:0] - index = 0 - pathlen = len(path) - while index < pathlen: - c = path[index:index+1] - if c == quote: # no expansion within single quotes - path = path[index + 1:] - pathlen = len(path) - try: - index = path.index(c) - res += c + path[:index + 1] - except ValueError: - res += c + path - index = pathlen - 1 - elif c == percent: # variable or '%' - if path[index + 1:index + 2] == percent: - res += c - index += 1 - else: - path = path[index+1:] - pathlen = len(path) - try: - index = path.index(percent) - except ValueError: - res += percent + path - index = pathlen - 1 - else: - var = path[:index] - try: - if environ is None: - value = os.fsencode(os.environ[os.fsdecode(var)]) - else: - value = environ[var] - except KeyError: - value = percent + var + percent - res += value - elif c == dollar: # variable or '$$' - if path[index + 1:index + 2] == dollar: - res += c - index += 1 - elif path[index + 1:index + 2] == brace: - path = path[index+2:] - pathlen = len(path) - try: - index = path.index(rbrace) - except ValueError: - res += dollar + brace + path - index = pathlen - 1 - else: - var = path[:index] - try: - if environ is None: - value = os.fsencode(os.environ[os.fsdecode(var)]) - else: - value = environ[var] - except KeyError: - value = dollar + brace + var + rbrace - res += value - else: - var = path[:0] - index += 1 - c = path[index:index + 1] - while c and c in varchars: - var += c - index += 1 - c = path[index:index + 1] - try: - if environ is None: - value = os.fsencode(os.environ[os.fsdecode(var)]) - else: - value = environ[var] - except KeyError: - value = dollar + var - res += value - if c: - index -= 1 + + def repl(m): + lastindex = m.lastindex + if lastindex is None: + return m[0] + name = m[lastindex] + if lastindex == 1: + if name == percent: + return name + if not name.endswith(percent): + return m[0] + name = name[:-1] else: - res += c - index += 1 - return res + if name == dollar: + return name + if name.startswith(brace): + if not name.endswith(rbrace): + return m[0] + name = name[1:-1] + + try: + if environ is None: + return os.fsencode(os.environ[os.fsdecode(name)]) + else: + return environ[name] + except KeyError: + return m[0] + + return sub(repl, path) # Normalize a path, e.g. A//B, A/./B and A/foo/../B all become A\B. diff --git a/Lib/posixpath.py b/Lib/posixpath.py index bfcf3c862f683d4..860064afeec031c 100644 --- a/Lib/posixpath.py +++ b/Lib/posixpath.py @@ -279,42 +279,41 @@ def expanduser(path): # This expands the forms $variable and ${variable} only. # Non-existent variables are left unchanged. -_varprog = None -_varprogb = None +_varpattern = r'\$(\w+|\{[^}]*\}?)' +_varsub = None +_varsubb = None def expandvars(path): """Expand shell variables of form $var and ${var}. Unknown variables are left unchanged.""" path = os.fspath(path) - global _varprog, _varprogb + global _varsub, _varsubb if isinstance(path, bytes): if b'$' not in path: return path - if not _varprogb: + if not _varsubb: import re - _varprogb = re.compile(br'\$(\w+|\{[^}]*\})', re.ASCII) - search = _varprogb.search + _varsubb = re.compile(_varpattern.encode(), re.ASCII).sub + sub = _varsubb start = b'{' end = b'}' environ = getattr(os, 'environb', None) else: if '$' not in path: return path - if not _varprog: + if not _varsub: import re - _varprog = re.compile(r'\$(\w+|\{[^}]*\})', re.ASCII) - search = _varprog.search + _varsub = re.compile(_varpattern, re.ASCII).sub + sub = _varsub start = '{' end = '}' environ = os.environ - i = 0 - while True: - m = search(path, i) - if not m: - break - i, j = m.span(0) - name = m.group(1) - if name.startswith(start) and name.endswith(end): + + def repl(m): + name = m[1] + if name.startswith(start): + if not name.endswith(end): + return m[0] name = name[1:-1] try: if environ is None: @@ -322,13 +321,11 @@ def expandvars(path): else: value = environ[name] except KeyError: - i = j + return m[0] else: - tail = path[j:] - path = path[:i] + value - i = len(path) - path += tail - return path + return value + + return sub(repl, path) # Normalize a path, e.g. A//B, A/./B and A/foo/../B all become A/B. diff --git a/Lib/test/test_genericpath.py b/Lib/test/test_genericpath.py index 8b291cda689ce1e..f5d42c136b0e618 100644 --- a/Lib/test/test_genericpath.py +++ b/Lib/test/test_genericpath.py @@ -9,7 +9,7 @@ import warnings from test import support from test.support.script_helper import assert_python_ok -from test.support import FakePath +from test.support import FakePath, EnvironmentVarGuard def create_file(filename, data=b'foo'): @@ -354,7 +354,7 @@ def test_expandvars(self): if self.pathmodule.__name__ == 'macpath': self.skipTest('macpath.expandvars is a stub') expandvars = self.pathmodule.expandvars - with support.EnvironmentVarGuard() as env: + with EnvironmentVarGuard() as env: env.clear() env["foo"] = "bar" env["{foo"] = "baz1" @@ -390,7 +390,7 @@ def test_expandvars_nonascii(self): expandvars = self.pathmodule.expandvars def check(value, expected): self.assertEqual(expandvars(value), expected) - with support.EnvironmentVarGuard() as env: + with EnvironmentVarGuard() as env: env.clear() nonascii = support.FS_NONASCII env['spam'] = nonascii @@ -411,6 +411,19 @@ def check(value, expected): os.fsencode('$bar%s bar' % nonascii)) check(b'$spam}bar', os.fsencode('%s}bar' % nonascii)) + @support.requires_resource('cpu') + def test_expandvars_large(self): + expandvars = self.pathmodule.expandvars + with EnvironmentVarGuard() as env: + env.clear() + env["A"] = "B" + n = 100_000 + self.assertEqual(expandvars('$A'*n), 'B'*n) + self.assertEqual(expandvars('${A}'*n), 'B'*n) + self.assertEqual(expandvars('$A!'*n), 'B!'*n) + self.assertEqual(expandvars('${A}A'*n), 'BA'*n) + self.assertEqual(expandvars('${'*10*n), '${'*10*n) + def test_abspath(self): self.assertIn("foo", self.pathmodule.abspath("foo")) with warnings.catch_warnings(): diff --git a/Lib/test/test_ntpath.py b/Lib/test/test_ntpath.py index 89b0d5303db5a3e..22169b13d46ceff 100644 --- a/Lib/test/test_ntpath.py +++ b/Lib/test/test_ntpath.py @@ -3,7 +3,7 @@ import sys import unittest import warnings -from test.support import TestFailed, FakePath +from test.support import TestFailed, FakePath, EnvironmentVarGuard from test import support, test_genericpath from tempfile import TemporaryFile @@ -210,7 +210,7 @@ def test_normpath(self): tester("ntpath.normpath('\\\\?\\D:/XY\\Z')", r'\\?\D:/XY\Z') def test_expandvars(self): - with support.EnvironmentVarGuard() as env: + with EnvironmentVarGuard() as env: env.clear() env["foo"] = "bar" env["{foo"] = "baz1" @@ -239,7 +239,7 @@ def test_expandvars(self): def test_expandvars_nonascii(self): def check(value, expected): tester('ntpath.expandvars(%r)' % value, expected) - with support.EnvironmentVarGuard() as env: + with EnvironmentVarGuard() as env: env.clear() nonascii = support.FS_NONASCII env['spam'] = nonascii @@ -255,10 +255,23 @@ def check(value, expected): check('%spam%bar', '%sbar' % nonascii) check('%{}%bar'.format(nonascii), 'ham%sbar' % nonascii) + @support.requires_resource('cpu') + def test_expandvars_large(self): + expandvars = ntpath.expandvars + with EnvironmentVarGuard() as env: + env.clear() + env["A"] = "B" + n = 100_000 + self.assertEqual(expandvars('%A%'*n), 'B'*n) + self.assertEqual(expandvars('%A%A'*n), 'BA'*n) + self.assertEqual(expandvars("''"*n + '%%'), "''"*n + '%') + self.assertEqual(expandvars("%%"*n), "%"*n) + self.assertEqual(expandvars("$$"*n), "$"*n) + def test_expanduser(self): tester('ntpath.expanduser("test")', 'test') - with support.EnvironmentVarGuard() as env: + with EnvironmentVarGuard() as env: env.clear() tester('ntpath.expanduser("~test")', '~test') @@ -469,6 +482,7 @@ def test_nt_helpers(self): self.assertIsInstance(b_final_path, bytes) self.assertGreater(len(b_final_path), 0) + class NtCommonTest(test_genericpath.CommonTest, unittest.TestCase): pathmodule = ntpath attributes = ['relpath'] diff --git a/Misc/NEWS.d/next/Security/2025-05-30-22-33-27.gh-issue-136065.bu337o.rst b/Misc/NEWS.d/next/Security/2025-05-30-22-33-27.gh-issue-136065.bu337o.rst new file mode 100644 index 000000000000000..1d152bb5318380b --- /dev/null +++ b/Misc/NEWS.d/next/Security/2025-05-30-22-33-27.gh-issue-136065.bu337o.rst @@ -0,0 +1 @@ +Fix quadratic complexity in :func:`os.path.expandvars`. From 752f27ddc042dc6335e43fcecf1b34cfd7373792 Mon Sep 17 00:00:00 2001 From: "Gregory P. Smith" <68491+gpshead@users.noreply.github.com> Date: Tue, 20 Jan 2026 14:51:43 -0800 Subject: [PATCH 038/129] [3.10] gh-143916: Reject control characters in wsgiref.headers.Headers gh-143916: Reject control characters in wsgiref.headers.Headers (GH-143917) * Add 'test.support' fixture for C0 control characters * gh-143916: Reject control characters in wsgiref.headers.Headers (cherry picked from commit f7fceed79ca1bceae8dbe5ba5bc8928564da7211) (cherry picked from commit 22e4d55285cee52bc4dbe061324e5f30bd4dee58) Co-authored-by: Seth Michael Larson (cherry picked from commit 2f840249550e082dc351743f474ba56da10478d2) (cherry picked from commit 9ee88250b4610311baff4daa1de9fe228f88f163) 3.7.17.x: test.support.control_characters_c0() has no list[str] annotation (not valid at runtime on 3.7). --- Lib/test/support/__init__.py | 7 +++++++ Lib/test/test_wsgiref.py | 11 +++++++++++ Lib/wsgiref/headers.py | 3 +++ .../2026-01-16-11-07-36.gh-issue-143916.dpWeOD.rst | 2 ++ 4 files changed, 23 insertions(+) create mode 100644 Misc/NEWS.d/next/Security/2026-01-16-11-07-36.gh-issue-143916.dpWeOD.rst diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index 6f4306516a3a26a..3a19d1d6f8d822f 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -3044,3 +3044,10 @@ def adjust_int_max_str_digits(max_digits): yield finally: sys.set_int_max_str_digits(current) + + +def control_characters_c0(): + """Returns a list of C0 control characters as strings. + C0 control characters defined as the byte range 0x00-0x1F, and 0x7F. + """ + return [chr(c) for c in range(0x00, 0x20)] + ["\x7F"] diff --git a/Lib/test/test_wsgiref.py b/Lib/test/test_wsgiref.py index da9481f839bf600..9032675971b3d5e 100644 --- a/Lib/test/test_wsgiref.py +++ b/Lib/test/test_wsgiref.py @@ -1,5 +1,6 @@ from unittest import mock from test import support +from test.support import control_characters_c0 from test.test_httpservers import NoLogRequestHandler from unittest import TestCase from wsgiref.util import setup_testing_defaults @@ -517,6 +518,16 @@ def testExtras(self): '\r\n' ) + def testRaisesControlCharacters(self): + headers = Headers() + for c0 in control_characters_c0(): + self.assertRaises(ValueError, headers.__setitem__, f"key{c0}", "val") + self.assertRaises(ValueError, headers.__setitem__, "key", f"val{c0}") + self.assertRaises(ValueError, headers.add_header, f"key{c0}", "val", param="param") + self.assertRaises(ValueError, headers.add_header, "key", f"val{c0}", param="param") + self.assertRaises(ValueError, headers.add_header, "key", "val", param=f"param{c0}") + + class ErrorHandler(BaseCGIHandler): """Simple handler subclass for testing BaseHandler""" diff --git a/Lib/wsgiref/headers.py b/Lib/wsgiref/headers.py index fab851c5a44430d..fd98e85d75492be 100644 --- a/Lib/wsgiref/headers.py +++ b/Lib/wsgiref/headers.py @@ -9,6 +9,7 @@ # existence of which force quoting of the parameter value. import re tspecials = re.compile(r'[ \(\)<>@,;:\\"/\[\]\?=]') +_control_chars_re = re.compile(r'[\x00-\x1F\x7F]') def _formatparam(param, value=None, quote=1): """Convenience function to format and return a key=value pair. @@ -41,6 +42,8 @@ def __init__(self, headers=None): def _convert_string_type(self, value): """Convert/check value type.""" if type(value) is str: + if _control_chars_re.search(value): + raise ValueError("Control characters not allowed in headers") return value raise AssertionError("Header names/values must be" " of type str (got {0})".format(repr(value))) diff --git a/Misc/NEWS.d/next/Security/2026-01-16-11-07-36.gh-issue-143916.dpWeOD.rst b/Misc/NEWS.d/next/Security/2026-01-16-11-07-36.gh-issue-143916.dpWeOD.rst new file mode 100644 index 000000000000000..44bd0b27059f944 --- /dev/null +++ b/Misc/NEWS.d/next/Security/2026-01-16-11-07-36.gh-issue-143916.dpWeOD.rst @@ -0,0 +1,2 @@ +Reject C0 control characters within wsgiref.headers.Headers fields, values, +and parameters. From a7400ead9fbc06bc217c6bd3db9eba60da8d4854 Mon Sep 17 00:00:00 2001 From: Seth Michael Larson Date: Sun, 25 Jan 2026 11:05:15 -0600 Subject: [PATCH 039/129] [3.10] gh-143925: Reject control characters in data: URL mediatypes (#144115) (cherry picked from commit f25509e78e8be6ea73c811ac2b8c928c28841b9f) (cherry picked from commit 2c9c746077d8119b5bcf5142316992e464594946) (cherry picked from commit 34d76b00dabde81a793bd06dd8ecb057838c4b38) (cherry picked from commit 922a106c3649e614e7b311bb68604bf4e56b1838) --- Lib/test/test_urllib.py | 8 ++++++++ Lib/urllib/request.py | 5 +++++ .../2026-01-16-11-51-19.gh-issue-143925.mrtcHW.rst | 1 + 3 files changed, 14 insertions(+) create mode 100644 Misc/NEWS.d/next/Security/2026-01-16-11-51-19.gh-issue-143925.mrtcHW.rst diff --git a/Lib/test/test_urllib.py b/Lib/test/test_urllib.py index 0305e7aa1884960..733ec21864ecfda 100644 --- a/Lib/test/test_urllib.py +++ b/Lib/test/test_urllib.py @@ -9,6 +9,7 @@ import unittest from unittest.mock import patch from test import support +from test.support import control_characters_c0 import os try: import ssl @@ -660,6 +661,13 @@ def test_invalid_base64_data(self): # missing padding character self.assertRaises(ValueError,urllib.request.urlopen,'data:;base64,Cg=') + def test_invalid_mediatype(self): + for c0 in control_characters_c0(): + self.assertRaises(ValueError,urllib.request.urlopen, + f'data:text/html;{c0},data') + for c0 in control_characters_c0(): + self.assertRaises(ValueError,urllib.request.urlopen, + f'data:text/html{c0};base64,ZGF0YQ==') class urlretrieve_FileTests(unittest.TestCase): """Test urllib.urlretrieve() on local files""" diff --git a/Lib/urllib/request.py b/Lib/urllib/request.py index 993fde87b9225dc..eac27a24117d0a3 100644 --- a/Lib/urllib/request.py +++ b/Lib/urllib/request.py @@ -1656,6 +1656,11 @@ def data_open(self, req): scheme, data = url.split(":",1) mediatype, data = data.split(",",1) + # Disallow control characters within mediatype. + if re.search(r"[\x00-\x1F\x7F]", mediatype): + raise ValueError( + "Control characters not allowed in data: mediatype") + # even base64 encoded data URLs might be quoted so unquote in any case: data = unquote_to_bytes(data) if mediatype.endswith(";base64"): diff --git a/Misc/NEWS.d/next/Security/2026-01-16-11-51-19.gh-issue-143925.mrtcHW.rst b/Misc/NEWS.d/next/Security/2026-01-16-11-51-19.gh-issue-143925.mrtcHW.rst new file mode 100644 index 000000000000000..46109dfbef3ee77 --- /dev/null +++ b/Misc/NEWS.d/next/Security/2026-01-16-11-51-19.gh-issue-143925.mrtcHW.rst @@ -0,0 +1 @@ +Reject control characters in ``data:`` URL media types. From add92c6f91bf6989b930c6ccae573a62937dbbe6 Mon Sep 17 00:00:00 2001 From: Seth Michael Larson Date: Sun, 25 Jan 2026 11:05:23 -0600 Subject: [PATCH 040/129] [3.10] gh-143919: Reject control characters in http cookies (#144094) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit (cherry picked from commit 95746b3a13a985787ef53b977129041971ed7f70) Co-authored-by: Bartosz Sławecki Co-authored-by: sobolevn (cherry picked from commit 7852d72b653fea0199acf5fc2a84f6f8b84eba8d) (cherry picked from commit 66da1a61b70b3575e0b390880b9437320d005882) --- Doc/library/http.cookies.rst | 4 +- Lib/http/cookies.py | 25 +++++++-- Lib/test/test_http_cookies.py | 52 +++++++++++++++++-- ...-01-16-11-13-15.gh-issue-143919.kchwZV.rst | 1 + 4 files changed, 73 insertions(+), 9 deletions(-) create mode 100644 Misc/NEWS.d/next/Security/2026-01-16-11-13-15.gh-issue-143919.kchwZV.rst diff --git a/Doc/library/http.cookies.rst b/Doc/library/http.cookies.rst index 01bedfd266a8d57..30190ee4cd463c0 100644 --- a/Doc/library/http.cookies.rst +++ b/Doc/library/http.cookies.rst @@ -262,9 +262,9 @@ The following example demonstrates how to use the :mod:`http.cookies` module. Set-Cookie: chips=ahoy Set-Cookie: vienna=finger >>> C = cookies.SimpleCookie() - >>> C.load('keebler="E=everybody; L=\\"Loves\\"; fudge=\\012;";') + >>> C.load('keebler="E=everybody; L=\\"Loves\\"; fudge=;";') >>> print(C) - Set-Cookie: keebler="E=everybody; L=\"Loves\"; fudge=\012;" + Set-Cookie: keebler="E=everybody; L=\"Loves\"; fudge=;" >>> C = cookies.SimpleCookie() >>> C["oreo"] = "doublestuff" >>> C["oreo"]["path"] = "/" diff --git a/Lib/http/cookies.py b/Lib/http/cookies.py index 3115b2c830e523c..31ade9f2ed94c3e 100644 --- a/Lib/http/cookies.py +++ b/Lib/http/cookies.py @@ -87,9 +87,9 @@ such trickeries do not confuse it. >>> C = cookies.SimpleCookie() - >>> C.load('keebler="E=everybody; L=\\"Loves\\"; fudge=\\012;";') + >>> C.load('keebler="E=everybody; L=\\"Loves\\"; fudge=;";') >>> print(C) - Set-Cookie: keebler="E=everybody; L=\"Loves\"; fudge=\012;" + Set-Cookie: keebler="E=everybody; L=\"Loves\"; fudge=;" Each element of the Cookie also supports all of the RFC 2109 Cookie attributes. Here's an example which sets the Path @@ -169,6 +169,15 @@ class CookieError(Exception): }) _is_legal_key = re.compile('[%s]+' % re.escape(_LegalChars)).fullmatch +_control_character_re = re.compile(r'[\x00-\x1F\x7F]') + + +def _has_control_character(*val): + """Detects control characters within a value. + Supports any type, as header values can be any type. + """ + return any(_control_character_re.search(str(v)) for v in val) + def _quote(str): r"""Quote a string for use in a cookie header. @@ -291,12 +300,16 @@ def __setitem__(self, K, V): K = K.lower() if not K in self._reserved: raise CookieError("Invalid attribute %r" % (K,)) + if _has_control_character(K, V): + raise CookieError(f"Control characters are not allowed in cookies {K!r} {V!r}") dict.__setitem__(self, K, V) def setdefault(self, key, val=None): key = key.lower() if key not in self._reserved: raise CookieError("Invalid attribute %r" % (key,)) + if _has_control_character(key, val): + raise CookieError("Control characters are not allowed in cookies %r %r" % (key, val,)) return dict.setdefault(self, key, val) def __eq__(self, morsel): @@ -332,6 +345,9 @@ def set(self, key, val, coded_val): raise CookieError('Attempt to set a reserved key %r' % (key,)) if not _is_legal_key(key): raise CookieError('Illegal key %r' % (key,)) + if _has_control_character(key, val, coded_val): + raise CookieError( + "Control characters are not allowed in cookies %r %r %r" % (key, val, coded_val,)) # It's a good key, so save it. self._key = key @@ -481,7 +497,10 @@ def output(self, attrs=None, header="Set-Cookie:", sep="\015\012"): result = [] items = sorted(self.items()) for key, value in items: - result.append(value.output(attrs, header)) + value_output = value.output(attrs, header) + if _has_control_character(value_output): + raise CookieError("Control characters are not allowed in cookies") + result.append(value_output) return sep.join(result) __str__ = output diff --git a/Lib/test/test_http_cookies.py b/Lib/test/test_http_cookies.py index fb39f9cea129dbc..659d547e84e8f8d 100644 --- a/Lib/test/test_http_cookies.py +++ b/Lib/test/test_http_cookies.py @@ -17,10 +17,10 @@ def test_basic(self): 'repr': "", 'output': 'Set-Cookie: chips=ahoy\nSet-Cookie: vienna=finger'}, - {'data': 'keebler="E=mc2; L=\\"Loves\\"; fudge=\\012;"', - 'dict': {'keebler' : 'E=mc2; L="Loves"; fudge=\012;'}, - 'repr': '''''', - 'output': 'Set-Cookie: keebler="E=mc2; L=\\"Loves\\"; fudge=\\012;"'}, + {'data': 'keebler="E=mc2; L=\\"Loves\\"; fudge=;"', + 'dict': {'keebler' : 'E=mc2; L="Loves"; fudge=;'}, + 'repr': '''''', + 'output': 'Set-Cookie: keebler="E=mc2; L=\\"Loves\\"; fudge=;"'}, # Check illegal cookies that have an '=' char in an unquoted value {'data': 'keebler=E=mc2', @@ -504,6 +504,50 @@ def test_repr(self): r'Set-Cookie: key=coded_val; ' r'expires=\w+, \d+ \w+ \d+ \d+:\d+:\d+ \w+') + def test_control_characters(self): + for c0 in support.control_characters_c0(): + morsel = cookies.Morsel() + + # .__setitem__() + with self.assertRaises(cookies.CookieError): + morsel[c0] = "val" + with self.assertRaises(cookies.CookieError): + morsel["path"] = c0 + + # .setdefault() + with self.assertRaises(cookies.CookieError): + morsel.setdefault("path", c0) + with self.assertRaises(cookies.CookieError): + morsel.setdefault(c0, "val") + + # .set() + with self.assertRaises(cookies.CookieError): + morsel.set(c0, "val", "coded-value") + with self.assertRaises(cookies.CookieError): + morsel.set("path", c0, "coded-value") + with self.assertRaises(cookies.CookieError): + morsel.set("path", "val", c0) + + def test_control_characters_output(self): + # Tests that even if the internals of Morsel are modified + # that a call to .output() has control character safeguards. + for c0 in support.control_characters_c0(): + morsel = cookies.Morsel() + morsel.set("key", "value", "coded-value") + morsel._key = c0 # Override private variable. + cookie = cookies.SimpleCookie() + cookie["cookie"] = morsel + with self.assertRaises(cookies.CookieError): + cookie.output() + + morsel = cookies.Morsel() + morsel.set("key", "value", "coded-value") + morsel._coded_value = c0 # Override private variable. + cookie = cookies.SimpleCookie() + cookie["cookie"] = morsel + with self.assertRaises(cookies.CookieError): + cookie.output() + def test_main(): run_unittest(CookieTests, MorselTests) run_doctest(cookies) diff --git a/Misc/NEWS.d/next/Security/2026-01-16-11-13-15.gh-issue-143919.kchwZV.rst b/Misc/NEWS.d/next/Security/2026-01-16-11-13-15.gh-issue-143919.kchwZV.rst new file mode 100644 index 000000000000000..788c3e4ac2ebf76 --- /dev/null +++ b/Misc/NEWS.d/next/Security/2026-01-16-11-13-15.gh-issue-143919.kchwZV.rst @@ -0,0 +1 @@ +Reject control characters in :class:`http.cookies.Morsel` fields and values. From b759ad09a3aa689c38c5f1f0c8cfa5f9a5ae77a4 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Sun, 25 Jan 2026 18:10:00 +0100 Subject: [PATCH 041/129] [3.10] gh-144125: email: verify headers are sound in BytesGenerator (#144180) gh-144125: email: verify headers are sound in BytesGenerator (cherry picked from commit 052e55e7d44718fe46cbba0ca995cb8fcc359413) Co-authored-by: Seth Michael Larson Co-authored-by: Denis Ledoux Co-authored-by: Denis Ledoux <5822488+beledouxdenis@users.noreply.github.com> Co-authored-by: Petr Viktorin <302922+encukou@users.noreply.github.com> Co-authored-by: Bas Bloemsaat <1586868+basbloemsaat@users.noreply.github.com> (cherry picked from commit 8cdf6204f4ae821f32993f8fc6bad0d318f95f36) (cherry picked from commit 3364febf1fc63791626c23d34140e44368bd8613) 3.7.17.x: bytes.removesuffix() (3.9+) replaced by slicing in BytesGenerator. --- Lib/email/generator.py | 12 +++++++++++- Lib/test/test_email/test_generator.py | 4 +++- Lib/test/test_email/test_policy.py | 6 +++++- .../2026-01-21-12-34-05.gh-issue-144125.TAz5uo.rst | 4 ++++ 4 files changed, 23 insertions(+), 3 deletions(-) create mode 100644 Misc/NEWS.d/next/Security/2026-01-21-12-34-05.gh-issue-144125.TAz5uo.rst diff --git a/Lib/email/generator.py b/Lib/email/generator.py index 27383c52b9f9ceb..e7f1739b78bd192 100644 --- a/Lib/email/generator.py +++ b/Lib/email/generator.py @@ -22,6 +22,7 @@ NLCRE = re.compile(r'\r\n|\r|\n') fcre = re.compile(r'^From ', re.MULTILINE) NEWLINE_WITHOUT_FWSP = re.compile(r'\r\n[^ \t]|\r[^ \n\t]|\n[^ \t]') +NEWLINE_WITHOUT_FWSP_BYTES = re.compile(br'\r\n[^ \t]|\r[^ \n\t]|\n[^ \t]') @@ -426,7 +427,16 @@ def _write_headers(self, msg): # This is almost the same as the string version, except for handling # strings with 8bit bytes. for h, v in msg.raw_items(): - self._fp.write(self.policy.fold_binary(h, v)) + folded = self.policy.fold_binary(h, v) + if self.policy.verify_generated_headers: + linesep = self.policy.linesep.encode() + if not folded.endswith(linesep): + raise HeaderWriteError( + f'folded header does not end with {linesep!r}: {folded!r}') + if NEWLINE_WITHOUT_FWSP_BYTES.search(folded[:len(folded) - len(linesep)]): + raise HeaderWriteError( + f'folded header contains newline: {folded!r}') + self._fp.write(folded) # A blank line always separates headers from body self.write(self._NL) diff --git a/Lib/test/test_email/test_generator.py b/Lib/test/test_email/test_generator.py index d29400f0ed1dbba..a641f871dd7d674 100644 --- a/Lib/test/test_email/test_generator.py +++ b/Lib/test/test_email/test_generator.py @@ -264,7 +264,7 @@ class TestGenerator(TestGeneratorBase, TestEmailBase): typ = str def test_verify_generated_headers(self): - """gh-121650: by default the generator prevents header injection""" + # gh-121650: by default the generator prevents header injection class LiteralHeader(str): name = 'Header' def fold(self, **kwargs): @@ -285,6 +285,8 @@ def fold(self, **kwargs): with self.assertRaises(email.errors.HeaderWriteError): message.as_string() + with self.assertRaises(email.errors.HeaderWriteError): + message.as_bytes() class TestBytesGenerator(TestGeneratorBase, TestEmailBase): diff --git a/Lib/test/test_email/test_policy.py b/Lib/test/test_email/test_policy.py index 1c08d7cd5ce1b39..838a6bb42febcb0 100644 --- a/Lib/test/test_email/test_policy.py +++ b/Lib/test/test_email/test_policy.py @@ -280,7 +280,7 @@ def test_short_maxlen_error(self): policy.fold("Subject", subject) def test_verify_generated_headers(self): - """Turning protection off allows header injection""" + # Turning protection off allows header injection policy = email.policy.default.clone(verify_generated_headers=False) for text in ( 'Header: Value\r\nBad: Injection\r\n', @@ -303,6 +303,10 @@ def fold(self, **kwargs): message.as_string(), f"{text}\nBody", ) + self.assertEqual( + message.as_bytes(), + f"{text}\nBody".encode(), + ) # XXX: Need subclassing tests. # For adding subclassed objects, make sure the usual rules apply (subclass diff --git a/Misc/NEWS.d/next/Security/2026-01-21-12-34-05.gh-issue-144125.TAz5uo.rst b/Misc/NEWS.d/next/Security/2026-01-21-12-34-05.gh-issue-144125.TAz5uo.rst new file mode 100644 index 000000000000000..e6333e724972c59 --- /dev/null +++ b/Misc/NEWS.d/next/Security/2026-01-21-12-34-05.gh-issue-144125.TAz5uo.rst @@ -0,0 +1,4 @@ +:mod:`~email.generator.BytesGenerator` will now refuse to serialize (write) headers +that are unsafely folded or delimited; see +:attr:`~email.policy.Policy.verify_generated_headers`. (Contributed by Bas +Bloemsaat and Petr Viktorin in :gh:`121650`). From 233f69f943a42c6b956833f7756db14afc246a5d Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Sun, 25 Jan 2026 18:10:42 +0100 Subject: [PATCH 042/129] [3.10] gh-119452: Fix a potential virtual memory allocation denial of service in http.server (GH-142216) (#142299) [3.14] gh-119452: Fix a potential virtual memory allocation denial of service in http.server (GH-142216) The CGI server on Windows could consume the amount of memory specified in the Content-Length header of the request even if the client does not send such much data. Now it reads the POST request body by chunks, therefore the memory consumption is proportional to the amount of sent data. (cherry picked from commit 0e4f4f1a4633f2d215fb5a803cae278aeea31845) Co-authored-by: Serhiy Storchaka (cherry picked from commit f2088a567e8b137581e00cd18c8d6b79973ab150) (cherry picked from commit 34072c545dc518928f072e5e6d703416ed00a252) 3.7.17.x: only the file7 CGI fixture is added (file5/file6 belong to tests 3.7 does not have). --- Lib/http/server.py | 17 ++++++++- Lib/test/test_httpservers.py | 38 +++++++++++++++++++ ...-05-23-11-44-41.gh-issue-119452.PRfsSv.rst | 5 +++ 3 files changed, 59 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Security/2024-05-23-11-44-41.gh-issue-119452.PRfsSv.rst diff --git a/Lib/http/server.py b/Lib/http/server.py index beabe3de7abb2e7..7106e3523b4a672 100644 --- a/Lib/http/server.py +++ b/Lib/http/server.py @@ -129,6 +129,10 @@ DEFAULT_ERROR_CONTENT_TYPE = "text/html;charset=utf-8" +# Data larger than this will be read in chunks, to prevent extreme +# overallocation. +_MIN_READ_BUF_SIZE = 1 << 20 + class HTTPServer(socketserver.TCPServer): allow_reuse_address = 1 # Seems to make sense in testing environment @@ -1213,7 +1217,18 @@ def run_cgi(self): env = env ) if self.command.lower() == "post" and nbytes > 0: - data = self.rfile.read(nbytes) + cursize = 0 + data = self.rfile.read(min(nbytes, _MIN_READ_BUF_SIZE)) + while len(data) < nbytes and len(data) != cursize: + cursize = len(data) + # This is a geometric increase in read size (never more + # than doubling out the current length of data per loop + # iteration). + delta = min(cursize, nbytes - cursize) + try: + data += self.rfile.read(delta) + except TimeoutError: + break else: data = None # throw away additional data [see bug #427345] diff --git a/Lib/test/test_httpservers.py b/Lib/test/test_httpservers.py index 8c9be6890039234..f2be621e7cc29d4 100644 --- a/Lib/test/test_httpservers.py +++ b/Lib/test/test_httpservers.py @@ -641,6 +641,20 @@ def test_html_escape_filename(self): print(os.environ["%s"]) """ +cgi_file7 = """\ +#!%s +import os +import sys + +print("Content-type: text/plain") +print() + +content_length = int(os.environ["CONTENT_LENGTH"]) +body = sys.stdin.buffer.read(content_length) + +print(f"{content_length} {len(body)}") +""" + @unittest.skipIf(hasattr(os, 'geteuid') and os.geteuid() == 0, "This test can't be run reliably as root (issue #13308).") @@ -663,6 +677,7 @@ def setUp(self): self.file2_path = None self.file3_path = None self.file4_path = None + self.file7_path = None # The shebang line should be pure ASCII: use symlink if possible. # See issue #7668. @@ -706,6 +721,11 @@ def setUp(self): file4.write(cgi_file4 % (self.pythonexe, 'QUERY_STRING')) os.chmod(self.file4_path, 0o777) + self.file7_path = os.path.join(self.cgi_dir, 'file7.py') + with open(self.file7_path, 'w', encoding='utf-8') as file7: + file7.write(cgi_file7 % self.pythonexe) + os.chmod(self.file7_path, 0o777) + os.chdir(self.parent_dir) def tearDown(self): @@ -723,6 +743,8 @@ def tearDown(self): os.remove(self.file3_path) if self.file4_path: os.remove(self.file4_path) + if self.file7_path: + os.remove(self.file7_path) os.rmdir(self.cgi_child_dir) os.rmdir(self.cgi_dir) os.rmdir(self.parent_dir) @@ -789,6 +811,22 @@ def test_post(self): self.assertEqual(res.read(), b'1, python, 123456' + self.linesep) + def test_large_content_length(self): + for w in range(15, 25): + size = 1 << w + body = b'X' * size + headers = {'Content-Length' : str(size)} + res = self.request('/cgi-bin/file7.py', 'POST', body, headers) + self.assertEqual(res.read(), b'%d %d' % (size, size) + self.linesep) + + def test_large_content_length_truncated(self): + with support.swap_attr(self.request_handler, 'timeout', 0.001): + for w in range(18, 65): + size = 1 << w + headers = {'Content-Length' : str(size)} + res = self.request('/cgi-bin/file1.py', 'POST', b'x', headers) + self.assertEqual(res.read(), b'Hello World' + self.linesep) + def test_invaliduri(self): res = self.request('/cgi-bin/invalid') res.read() diff --git a/Misc/NEWS.d/next/Security/2024-05-23-11-44-41.gh-issue-119452.PRfsSv.rst b/Misc/NEWS.d/next/Security/2024-05-23-11-44-41.gh-issue-119452.PRfsSv.rst new file mode 100644 index 000000000000000..98956627f2b30df --- /dev/null +++ b/Misc/NEWS.d/next/Security/2024-05-23-11-44-41.gh-issue-119452.PRfsSv.rst @@ -0,0 +1,5 @@ +Fix a potential memory denial of service in the :mod:`http.server` module. +When a malicious user is connected to the CGI server on Windows, it could cause +an arbitrary amount of memory to be allocated. +This could have led to symptoms including a :exc:`MemoryError`, swapping, out +of memory (OOM) killed processes or containers, or even system crashes. From 2b8012ff777e21ca96eaeb1102c1622d24712143 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Sun, 25 Jan 2026 18:10:49 +0100 Subject: [PATCH 043/129] [3.10] gh-142145: Remove quadratic behavior in node ID cache clearing (GH-142146) (#142213) * gh-142145: Remove quadratic behavior in node ID cache clearing (GH-142146) * Remove quadratic behavior in node ID cache clearing Co-authored-by: Jacob Walls <38668450+jacobtylerwalls@users.noreply.github.com> * Add news fragment --------- (cherry picked from commit 08d8e18ad81cd45bc4a27d6da478b51ea49486e4) Co-authored-by: Seth Michael Larson Co-authored-by: Jacob Walls <38668450+jacobtylerwalls@users.noreply.github.com> * [3.14] gh-142754: Ensure that Element & Attr instances have the ownerDocument attribute (GH-142794) (#142818) gh-142754: Ensure that Element & Attr instances have the ownerDocument attribute (GH-142794) (cherry picked from commit 1cc7551b3f9f71efbc88d96dce90f82de98b2454) Co-authored-by: Petr Viktorin Co-authored-by: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> * gh-142145: relax the no-longer-quadratic test timing (GH-143030) * gh-142145: relax the no-longer-quadratic test timing * require cpu resource (cherry picked from commit 8d2d7bb2e754f8649a68ce4116271a4932f76907) Co-authored-by: Gregory P. Smith <68491+gpshead@users.noreply.github.com> * merge NEWS entries into one --------- Co-authored-by: Seth Michael Larson Co-authored-by: Jacob Walls <38668450+jacobtylerwalls@users.noreply.github.com> Co-authored-by: Petr Viktorin Co-authored-by: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> Co-authored-by: Gregory P. Smith <68491+gpshead@users.noreply.github.com> Co-authored-by: Gregory P. Smith (cherry picked from commit c97e87593063d84a2bd9fe7068b30eb44de23dc0) (cherry picked from commit a608df2354879e5a71f1cac0fa52e0b23b6201e7) --- Lib/test/test_minidom.py | 34 ++++++++++++++++++- Lib/xml/dom/minidom.py | 11 ++---- ...-12-01-09-36-45.gh-issue-142145.tcAUhg.rst | 6 ++++ 3 files changed, 42 insertions(+), 9 deletions(-) create mode 100644 Misc/NEWS.d/next/Security/2025-12-01-09-36-45.gh-issue-142145.tcAUhg.rst diff --git a/Lib/test/test_minidom.py b/Lib/test/test_minidom.py index 0b76f6e87bc50b5..2cde54d4187275f 100644 --- a/Lib/test/test_minidom.py +++ b/Lib/test/test_minidom.py @@ -2,13 +2,15 @@ import copy import pickle +import time +import io from test import support import unittest import pyexpat import xml.dom.minidom -from xml.dom.minidom import parse, Node, Document, parseString +from xml.dom.minidom import parse, Attr, Node, Document, Element, parseString from xml.dom.minidom import getDOMImplementation from xml.parsers.expat import ExpatError @@ -162,6 +164,36 @@ def testAppendChild(self): self.confirm(dom.documentElement.childNodes[-1].data == "Hello") dom.unlink() + @support.requires_resource('cpu') + def testAppendChildNoQuadraticComplexity(self): + impl = getDOMImplementation() + + newdoc = impl.createDocument(None, "some_tag", None) + top_element = newdoc.documentElement + children = [newdoc.createElement(f"child-{i}") for i in range(1, 2 ** 15 + 1)] + element = top_element + + start = time.monotonic() + for child in children: + element.appendChild(child) + element = child + end = time.monotonic() + + # This example used to take at least 30 seconds. + # Conservative assertion due to the wide variety of systems and + # build configs timing based tests wind up run under. + # A --with-address-sanitizer --with-pydebug build on a rpi5 still + # completes this loop in <0.5 seconds. + self.assertLess(end - start, 4) + + def testSetAttributeNodeWithoutOwnerDocument(self): + # regression test for gh-142754 + elem = Element("test") + attr = Attr("id") + attr.value = "test-id" + elem.setAttributeNode(attr) + self.assertEqual(elem.getAttribute("id"), "test-id") + def testAppendChildFragment(self): dom, orig, c1, c2, c3, frag = self._create_fragment_test_nodes() dom.documentElement.appendChild(frag) diff --git a/Lib/xml/dom/minidom.py b/Lib/xml/dom/minidom.py index 24957ea14fa73b1..600bf0b781dcdaa 100644 --- a/Lib/xml/dom/minidom.py +++ b/Lib/xml/dom/minidom.py @@ -291,13 +291,6 @@ def _append_child(self, node): childNodes.append(node) node.parentNode = self -def _in_document(node): - # return True iff node is part of a document tree - while node is not None: - if node.nodeType == Node.DOCUMENT_NODE: - return True - node = node.parentNode - return False def _write_data(writer, data): "Writes datachars to writer." @@ -354,6 +347,7 @@ class Attr(Node): def __init__(self, qName, namespaceURI=EMPTY_NAMESPACE, localName=None, prefix=None): self.ownerElement = None + self.ownerDocument = None self._name = qName self.namespaceURI = namespaceURI self._prefix = prefix @@ -677,6 +671,7 @@ class Element(Node): def __init__(self, tagName, namespaceURI=EMPTY_NAMESPACE, prefix=None, localName=None): + self.ownerDocument = None self.parentNode = None self.tagName = self.nodeName = tagName self.prefix = prefix @@ -1512,7 +1507,7 @@ def _clear_id_cache(node): if node.nodeType == Node.DOCUMENT_NODE: node._id_cache.clear() node._id_search_stack = None - elif _in_document(node): + elif node.ownerDocument: node.ownerDocument._id_cache.clear() node.ownerDocument._id_search_stack= None diff --git a/Misc/NEWS.d/next/Security/2025-12-01-09-36-45.gh-issue-142145.tcAUhg.rst b/Misc/NEWS.d/next/Security/2025-12-01-09-36-45.gh-issue-142145.tcAUhg.rst new file mode 100644 index 000000000000000..05c7df35d14bef1 --- /dev/null +++ b/Misc/NEWS.d/next/Security/2025-12-01-09-36-45.gh-issue-142145.tcAUhg.rst @@ -0,0 +1,6 @@ +Remove quadratic behavior in ``xml.minidom`` node ID cache clearing. In order +to do this without breaking existing users, we also add the *ownerDocument* +attribute to :mod:`xml.dom.minidom` elements and attributes created by directly +instantiating the ``Element`` or ``Attr`` class. Note that this way of creating +nodes is not supported; creator functions like +:py:meth:`xml.dom.Document.documentElement` should be used instead. From 567a59c38f139d076b7b8692cf071b537ec0b584 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Sun, 25 Jan 2026 18:10:57 +0100 Subject: [PATCH 044/129] [3.10] gh-119451: Fix a potential denial of service in http.client (GH-119454) (#142142) gh-119451: Fix a potential denial of service in http.client (GH-119454) Reading the whole body of the HTTP response could cause OOM if the Content-Length value is too large even if the server does not send a large amount of data. Now the HTTP client reads large data by chunks, therefore the amount of consumed memory is proportional to the amount of sent data. (cherry picked from commit 5a4c4a033a4a54481be6870aa1896fad732555b5) Co-authored-by: Serhiy Storchaka (cherry picked from commit 5dc101675fd22918facbbe0fecdc821502beaaf0) (cherry picked from commit 3b80a0501480bd9506068dc0b0c9c7f71c5fcf6e) 3.7.17.x: tests listen with socket.socket() + bind() + listen() (no socket.create_server() in 3.7). --- Lib/http/client.py | 32 ++++++--- Lib/test/test_httplib.py | 70 +++++++++++++++++++ ...-05-23-11-47-48.gh-issue-119451.qkJe9-.rst | 5 ++ 3 files changed, 99 insertions(+), 8 deletions(-) create mode 100644 Misc/NEWS.d/next/Security/2024-05-23-11-47-48.gh-issue-119451.qkJe9-.rst diff --git a/Lib/http/client.py b/Lib/http/client.py index b756f607d344ef7..640019153456778 100644 --- a/Lib/http/client.py +++ b/Lib/http/client.py @@ -112,6 +112,11 @@ _MAXLINE = 65536 _MAXHEADERS = 100 +# Data larger than this will be read in chunks, to prevent extreme +# overallocation. +_MIN_READ_BUF_SIZE = 1 << 20 + + # Header name/value ABNF (http://tools.ietf.org/html/rfc7230#section-3.2) # # VCHAR = %x21-7E @@ -623,14 +628,25 @@ def _safe_read(self, amt): reading. If the bytes are truly not available (due to EOF), then the IncompleteRead exception can be used to detect the problem. """ - s = [] - while amt > 0: - chunk = self.fp.read(min(amt, MAXAMOUNT)) - if not chunk: - raise IncompleteRead(b''.join(s), amt) - s.append(chunk) - amt -= len(chunk) - return b"".join(s) + cursize = min(amt, _MIN_READ_BUF_SIZE) + data = self.fp.read(cursize) + if len(data) >= amt: + return data + if len(data) < cursize: + raise IncompleteRead(data, amt - len(data)) + + data = io.BytesIO(data) + data.seek(0, 2) + while True: + # This is a geometric increase in read size (never more than + # doubling out the current length of data per loop iteration). + delta = min(cursize, amt - cursize) + data.write(self.fp.read(delta)) + if data.tell() >= amt: + return data.getvalue() + cursize += delta + if data.tell() < cursize: + raise IncompleteRead(data.getvalue(), amt - data.tell()) def _safe_readinto(self, b): """Same as _safe_read, but for reading into a buffer.""" diff --git a/Lib/test/test_httplib.py b/Lib/test/test_httplib.py index a65044eea60dc05..6f24bca4d7b7d37 100644 --- a/Lib/test/test_httplib.py +++ b/Lib/test/test_httplib.py @@ -1192,6 +1192,76 @@ def run_server(): thread.join() self.assertEqual(result, b"proxied data\n") + def test_large_content_length(self): + serv = socket.socket() + serv.bind((HOST, 0)) + serv.listen() + self.addCleanup(serv.close) + + def run_server(): + [conn, address] = serv.accept() + with conn: + while conn.recv(1024): + conn.sendall( + b"HTTP/1.1 200 Ok\r\n" + b"Content-Length: %d\r\n" + b"\r\n" % size) + conn.sendall(b'A' * (size//3)) + conn.sendall(b'B' * (size - size//3)) + + thread = threading.Thread(target=run_server) + thread.start() + self.addCleanup(thread.join, 1.0) + + conn = client.HTTPConnection(*serv.getsockname()) + try: + for w in range(15, 27): + size = 1 << w + conn.request("GET", "/") + with conn.getresponse() as response: + self.assertEqual(len(response.read()), size) + finally: + conn.close() + thread.join(1.0) + + def test_large_content_length_truncated(self): + serv = socket.socket() + serv.bind((HOST, 0)) + serv.listen() + self.addCleanup(serv.close) + + def run_server(): + while True: + [conn, address] = serv.accept() + with conn: + conn.recv(1024) + if not size: + break + conn.sendall( + b"HTTP/1.1 200 Ok\r\n" + b"Content-Length: %d\r\n" + b"\r\n" + b"Text" % size) + + thread = threading.Thread(target=run_server) + thread.start() + self.addCleanup(thread.join, 1.0) + + conn = client.HTTPConnection(*serv.getsockname()) + try: + for w in range(18, 65): + size = 1 << w + conn.request("GET", "/") + with conn.getresponse() as response: + self.assertRaises(client.IncompleteRead, response.read) + conn.close() + finally: + conn.close() + size = 0 + conn.request("GET", "/") + conn.close() + thread.join(1.0) + def test_putrequest_override_domain_validation(self): """ It should be possible to override the default validation diff --git a/Misc/NEWS.d/next/Security/2024-05-23-11-47-48.gh-issue-119451.qkJe9-.rst b/Misc/NEWS.d/next/Security/2024-05-23-11-47-48.gh-issue-119451.qkJe9-.rst new file mode 100644 index 000000000000000..6d6f25cd2f8bf7c --- /dev/null +++ b/Misc/NEWS.d/next/Security/2024-05-23-11-47-48.gh-issue-119451.qkJe9-.rst @@ -0,0 +1,5 @@ +Fix a potential memory denial of service in the :mod:`http.client` module. +When connecting to a malicious server, it could cause +an arbitrary amount of memory to be allocated. +This could have led to symptoms including a :exc:`MemoryError`, swapping, out +of memory (OOM) killed processes or containers, or even system crashes. From 76668b2e052bbe50d5c5877b3d64f4e1f1b3f140 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Tue, 3 Mar 2026 00:54:18 +0200 Subject: [PATCH 045/129] [3.10] gh-119342: Fix a potential denial of service in plistlib (GH-119343) (#142151) Reading a specially prepared small Plist file could cause OOM because file's read(n) preallocates a bytes object for reading the specified amount of data. Now plistlib reads large data by chunks, therefore the upper limit of consumed memory is proportional to the size of the input file. (cherry picked from commit 694922cf40aa3a28f898b5f5ee08b71b4922df70) (cherry picked from commit 568342cfc8f002d9a15f30238f26b9d2e0e79036) (cherry picked from commit 7ca4518540b6d51982c894478b8dedc1bd9b0fec) 3.7.17.x: 3.7 still wraps data in plistlib.Data when use_builtin_types is false; test uses test.support.TESTFN/unlink. --- Lib/plistlib.py | 31 ++++++++++------ Lib/test/test_plistlib.py | 37 +++++++++++++++++-- ...-05-21-22-11-31.gh-issue-119342.BTFj4Z.rst | 5 +++ 3 files changed, 59 insertions(+), 14 deletions(-) create mode 100644 Misc/NEWS.d/next/Security/2024-05-21-22-11-31.gh-issue-119342.BTFj4Z.rst diff --git a/Lib/plistlib.py b/Lib/plistlib.py index dd17e3b4d81f124..86ebfd7b164cc5a 100644 --- a/Lib/plistlib.py +++ b/Lib/plistlib.py @@ -68,6 +68,9 @@ PlistFormat = enum.Enum('PlistFormat', 'FMT_XML FMT_BINARY', module=__name__) globals().update(PlistFormat.__members__) +# Data larger than this will be read in chunks, to prevent extreme +# overallocation. +_MIN_READ_BUF_SIZE = 1 << 20 # # @@ -581,12 +584,24 @@ def _get_size(self, tokenL): return tokenL + def _read(self, size): + cursize = min(size, _MIN_READ_BUF_SIZE) + data = self._fp.read(cursize) + while True: + if len(data) != cursize: + raise InvalidFileException + if cursize == size: + return data + delta = min(cursize, size - cursize) + data += self._fp.read(delta) + cursize += delta + def _read_ints(self, n, size): - data = self._fp.read(size * n) + data = self._read(size * n) if size in _BINARY_FORMAT: return struct.unpack(f'>{n}{_BINARY_FORMAT[size]}', data) else: - if not size or len(data) != size * n: + if not size: raise InvalidFileException() return tuple(int.from_bytes(data[i: i + size], 'big') for i in range(0, size * n, size)) @@ -643,24 +658,18 @@ def _read_object(self, ref): elif tokenH == 0x40: # data s = self._get_size(tokenL) - result = self._fp.read(s) - if len(result) != s: - raise InvalidFileException() + result = self._read(s) if not self._use_builtin_types: result = Data(result) elif tokenH == 0x50: # ascii string s = self._get_size(tokenL) - data = self._fp.read(s) - if len(data) != s: - raise InvalidFileException() + data = self._read(s) result = data.decode('ascii') elif tokenH == 0x60: # unicode string s = self._get_size(tokenL) * 2 - data = self._fp.read(s) - if len(data) != s: - raise InvalidFileException() + data = self._read(s) result = data.decode('utf-16be') # tokenH == 0x80 is documented as 'UID' and appears to be used for diff --git a/Lib/test/test_plistlib.py b/Lib/test/test_plistlib.py index c6c95fe67ff71ab..1e7dc8a9a8c3a41 100644 --- a/Lib/test/test_plistlib.py +++ b/Lib/test/test_plistlib.py @@ -744,8 +744,7 @@ def test_xml_plist_with_entity_decl(self): class TestBinaryPlistlib(unittest.TestCase): - @staticmethod - def decode(*objects, offset_size=1, ref_size=1): + def build(self, *objects, offset_size=1, ref_size=1): data = [b'bplist00'] offset = 8 offsets = [] @@ -757,7 +756,11 @@ def decode(*objects, offset_size=1, ref_size=1): len(objects), 0, offset) data.extend(offsets) data.append(tail) - return plistlib.loads(b''.join(data), fmt=plistlib.FMT_BINARY) + return b''.join(data) + + def decode(self, *objects, offset_size=1, ref_size=1): + data = self.build(*objects, offset_size=offset_size, ref_size=ref_size) + return plistlib.loads(data, fmt=plistlib.FMT_BINARY) def test_nonstandard_refs_size(self): # Issue #21538: Refs and offsets are 24-bit integers @@ -867,6 +870,34 @@ def test_invalid_binary(self): with self.assertRaises(plistlib.InvalidFileException): plistlib.loads(b'bplist00' + data, fmt=plistlib.FMT_BINARY) + def test_truncated_large_data(self): + self.addCleanup(support.unlink, support.TESTFN) + def check(data): + with open(support.TESTFN, 'wb') as f: + f.write(data) + # buffered file + with open(support.TESTFN, 'rb') as f: + with self.assertRaises(plistlib.InvalidFileException): + plistlib.load(f, fmt=plistlib.FMT_BINARY) + # unbuffered file + with open(support.TESTFN, 'rb', buffering=0) as f: + with self.assertRaises(plistlib.InvalidFileException): + plistlib.load(f, fmt=plistlib.FMT_BINARY) + for w in range(20, 64): + s = 1 << w + # data + check(self.build(b'\x4f\x13' + s.to_bytes(8, 'big'))) + # ascii string + check(self.build(b'\x5f\x13' + s.to_bytes(8, 'big'))) + # unicode string + check(self.build(b'\x6f\x13' + s.to_bytes(8, 'big'))) + # array + check(self.build(b'\xaf\x13' + s.to_bytes(8, 'big'))) + # dict + check(self.build(b'\xdf\x13' + s.to_bytes(8, 'big'))) + # number of objects + check(b'bplist00' + struct.pack('>6xBBQQQ', 1, 1, s, 0, 8)) + class TestPlistlibDeprecated(unittest.TestCase): def test_io_deprecated(self): diff --git a/Misc/NEWS.d/next/Security/2024-05-21-22-11-31.gh-issue-119342.BTFj4Z.rst b/Misc/NEWS.d/next/Security/2024-05-21-22-11-31.gh-issue-119342.BTFj4Z.rst new file mode 100644 index 000000000000000..04fd8faca4cf7e0 --- /dev/null +++ b/Misc/NEWS.d/next/Security/2024-05-21-22-11-31.gh-issue-119342.BTFj4Z.rst @@ -0,0 +1,5 @@ +Fix a potential memory denial of service in the :mod:`plistlib` module. +When reading a Plist file received from untrusted source, it could cause +an arbitrary amount of memory to be allocated. +This could have led to symptoms including a :exc:`MemoryError`, swapping, out +of memory (OOM) killed processes or containers, or even system crashes. From 84df321ea8ce0032b5d1b86b109ef0927d8285e8 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Mon, 2 Mar 2026 23:55:00 +0100 Subject: [PATCH 046/129] [3.10] gh-144833: Fix use-after-free in SSL module when SSL_new() fails (GH-144843) (#144862) gh-144833: Fix use-after-free in SSL module when SSL_new() fails (GH-144843) In newPySSLSocket(), when SSL_new() returns NULL, Py_DECREF(self) was called before _setSSLError(get_state_ctx(self), ...), causing a use-after-free. Additionally, get_state_ctx() was called with self (PySSLSocket*) instead of sslctx (PySSLContext*), which is a type confusion bug. Fix by calling _setSSLError() before Py_DECREF() and using sslctx instead of self for get_state_ctx(). (cherry picked from commit c91638ca0671b8038831f963ed44e66cdda006a2) Co-authored-by: Ramin Farajpour Cami (cherry picked from commit 4be624a99472e5540825ebbda2a3e22bb3893e24) (cherry picked from commit b04c71ba672854639a36a2147250f63270c844cd) --- .../Library/2026-02-15-00-00-00.gh-issue-144833.TUelo1.rst | 3 +++ Modules/_ssl.c | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Library/2026-02-15-00-00-00.gh-issue-144833.TUelo1.rst diff --git a/Misc/NEWS.d/next/Library/2026-02-15-00-00-00.gh-issue-144833.TUelo1.rst b/Misc/NEWS.d/next/Library/2026-02-15-00-00-00.gh-issue-144833.TUelo1.rst new file mode 100644 index 000000000000000..6d5b18f59ee7ea5 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-02-15-00-00-00.gh-issue-144833.TUelo1.rst @@ -0,0 +1,3 @@ +Fixed a use-after-free in :mod:`ssl` when ``SSL_new()`` returns NULL in +``newPySSLSocket()``. The error was reported via a dangling pointer after the +object had already been freed. diff --git a/Modules/_ssl.c b/Modules/_ssl.c index 1458888d20ce6d6..cdff4ebbbe9e26a 100644 --- a/Modules/_ssl.c +++ b/Modules/_ssl.c @@ -942,8 +942,8 @@ newPySSLSocket(PySSLContext *sslctx, PySocketSockObject *sock, self->ssl = SSL_new(ctx); PySSL_END_ALLOW_THREADS if (self->ssl == NULL) { - Py_DECREF(self); _setSSLError(NULL, 0, __FILE__, __LINE__); + Py_DECREF(self); return NULL; } SSL_set_app_data(self->ssl, self); From a5342bc78fd7c61ceabbaca390ec8c6c7e515272 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Mon, 2 Mar 2026 23:59:29 +0100 Subject: [PATCH 047/129] [3.10] gh-143916: Allow HTAB in wsgiref header values (#145140) gh-143916: Allow HTAB in wsgiref header values (cherry picked from commit 66da7bf6fe7b81e3ecc9c0a25bd47d4616c8d1a6) Co-authored-by: Seth Michael Larson Co-authored-by: Victor Stinner (cherry picked from commit c592227ffb48679af9845a45dbb0875d975bb219) (cherry picked from commit 32ec6c3b4dd1c601dd6279390b455578e36393be) --- Lib/test/test_wsgiref.py | 20 +++++++++++++------- Lib/wsgiref/headers.py | 35 ++++++++++++++++++++--------------- 2 files changed, 33 insertions(+), 22 deletions(-) diff --git a/Lib/test/test_wsgiref.py b/Lib/test/test_wsgiref.py index 9032675971b3d5e..edacb1e7160ba8d 100644 --- a/Lib/test/test_wsgiref.py +++ b/Lib/test/test_wsgiref.py @@ -519,14 +519,20 @@ def testExtras(self): ) def testRaisesControlCharacters(self): - headers = Headers() for c0 in control_characters_c0(): - self.assertRaises(ValueError, headers.__setitem__, f"key{c0}", "val") - self.assertRaises(ValueError, headers.__setitem__, "key", f"val{c0}") - self.assertRaises(ValueError, headers.add_header, f"key{c0}", "val", param="param") - self.assertRaises(ValueError, headers.add_header, "key", f"val{c0}", param="param") - self.assertRaises(ValueError, headers.add_header, "key", "val", param=f"param{c0}") - + with self.subTest(c0): + headers = Headers() + self.assertRaises(ValueError, headers.__setitem__, f"key{c0}", "val") + self.assertRaises(ValueError, headers.add_header, f"key{c0}", "val", param="param") + # HTAB (\x09) is allowed in values, not names. + if c0 == "\t": + headers["key"] = f"val{c0}" + headers.add_header("key", f"val{c0}") + headers.setdefault(f"key", f"val{c0}") + else: + self.assertRaises(ValueError, headers.__setitem__, "key", f"val{c0}") + self.assertRaises(ValueError, headers.add_header, "key", f"val{c0}", param="param") + self.assertRaises(ValueError, headers.add_header, "key", "val", param=f"param{c0}") class ErrorHandler(BaseCGIHandler): """Simple handler subclass for testing BaseHandler""" diff --git a/Lib/wsgiref/headers.py b/Lib/wsgiref/headers.py index fd98e85d75492be..17559b0a37bd201 100644 --- a/Lib/wsgiref/headers.py +++ b/Lib/wsgiref/headers.py @@ -9,7 +9,11 @@ # existence of which force quoting of the parameter value. import re tspecials = re.compile(r'[ \(\)<>@,;:\\"/\[\]\?=]') -_control_chars_re = re.compile(r'[\x00-\x1F\x7F]') +# Disallowed characters for headers and values. +# HTAB (\x09) is allowed in header values, but +# not in header names. (RFC 9110 Section 5.5) +_name_disallowed_re = re.compile(r'[\x00-\x1F\x7F]') +_value_disallowed_re = re.compile(r'[\x00-\x08\x0A-\x1F\x7F]') def _formatparam(param, value=None, quote=1): """Convenience function to format and return a key=value pair. @@ -36,13 +40,14 @@ def __init__(self, headers=None): self._headers = headers if __debug__: for k, v in headers: - self._convert_string_type(k) - self._convert_string_type(v) + self._convert_string_type(k, name=True) + self._convert_string_type(v, name=False) - def _convert_string_type(self, value): + def _convert_string_type(self, value, *, name): """Convert/check value type.""" if type(value) is str: - if _control_chars_re.search(value): + regex = (_name_disallowed_re if name else _value_disallowed_re) + if regex.search(value): raise ValueError("Control characters not allowed in headers") return value raise AssertionError("Header names/values must be" @@ -56,14 +61,14 @@ def __setitem__(self, name, val): """Set the value of a header.""" del self[name] self._headers.append( - (self._convert_string_type(name), self._convert_string_type(val))) + (self._convert_string_type(name, name=True), self._convert_string_type(val, name=False))) def __delitem__(self,name): """Delete all occurrences of a header, if present. Does *not* raise an exception if the header is missing. """ - name = self._convert_string_type(name.lower()) + name = self._convert_string_type(name.lower(), name=True) self._headers[:] = [kv for kv in self._headers if kv[0].lower() != name] def __getitem__(self,name): @@ -90,13 +95,13 @@ def get_all(self, name): fields deleted and re-inserted are always appended to the header list. If no fields exist with the given name, returns an empty list. """ - name = self._convert_string_type(name.lower()) + name = self._convert_string_type(name.lower(), name=True) return [kv[1] for kv in self._headers if kv[0].lower()==name] def get(self,name,default=None): """Get the first header value for 'name', or return 'default'""" - name = self._convert_string_type(name.lower()) + name = self._convert_string_type(name.lower(), name=True) for k,v in self._headers: if k.lower()==name: return v @@ -151,8 +156,8 @@ def setdefault(self,name,value): and value 'value'.""" result = self.get(name) if result is None: - self._headers.append((self._convert_string_type(name), - self._convert_string_type(value))) + self._headers.append((self._convert_string_type(name, name=True), + self._convert_string_type(value, name=False))) return value else: return result @@ -175,13 +180,13 @@ def add_header(self, _name, _value, **_params): """ parts = [] if _value is not None: - _value = self._convert_string_type(_value) + _value = self._convert_string_type(_value, name=False) parts.append(_value) for k, v in _params.items(): - k = self._convert_string_type(k) + k = self._convert_string_type(k, name=True) if v is None: parts.append(k.replace('_', '-')) else: - v = self._convert_string_type(v) + v = self._convert_string_type(v, name=False) parts.append(_formatparam(k.replace('_', '-'), v)) - self._headers.append((self._convert_string_type(_name), "; ".join(parts))) + self._headers.append((self._convert_string_type(_name, name=True), "; ".join(parts))) From a124829543a176ce0b10b5b2abee69c490e86ff2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edgar=20Ram=C3=ADrez=20Mondrag=C3=B3n?= Date: Mon, 2 Mar 2026 17:58:52 -0600 Subject: [PATCH 048/129] [3.10] gh-143935: Email preserve parens when folding comments (GH-143936) (#144350) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fix a bug in the folding of comments when flattening an email message using a modern email policy. Comments consisting of a very long sequence of non-foldable characters could trigger a forced line wrap that omitted the required leading space on the continuation line, causing the remainder of the comment to be interpreted as a new header field. This enabled header injection with carefully crafted inputs. (cherry picked from commit 17d1490) Co-authored-by: Seth Michael Larson seth@python.org Co-authored-by: Denis Ledoux dle@odoo.com - Issue: Fix folding of long comments of unfoldable characters in email headers #143935 Signed-off-by: Edgar Ramírez Mondragón Co-authored-by: Seth Michael Larson (cherry picked from commit 003b8315669b9f08b1010a49071f73f15f818094) (cherry picked from commit dd0809a627addc18befb781ebac8969c96d7f078) --- Lib/email/_header_value_parser.py | 15 +++++++++++- .../test_email/test__header_value_parser.py | 23 +++++++++++++++++++ ...-01-16-14-40-31.gh-issue-143935.U2YtKl.rst | 6 +++++ 3 files changed, 43 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Security/2026-01-16-14-40-31.gh-issue-143935.U2YtKl.rst diff --git a/Lib/email/_header_value_parser.py b/Lib/email/_header_value_parser.py index d73360ed6fa1e08..c83dc5554fd78e0 100644 --- a/Lib/email/_header_value_parser.py +++ b/Lib/email/_header_value_parser.py @@ -102,6 +102,12 @@ def make_quoted_pairs(value): return str(value).replace('\\', '\\\\').replace('"', '\\"') +def make_parenthesis_pairs(value): + """Escape parenthesis and backslash for use within a comment.""" + return str(value).replace('\\', '\\\\') \ + .replace('(', '\\(').replace(')', '\\)') + + def quote_string(value): escaped = make_quoted_pairs(value) return f'"{escaped}"' @@ -920,7 +926,7 @@ def value(self): return ' ' def startswith_fws(self): - return True + return self and self[0] in WSP class ValueTerminal(Terminal): @@ -2746,6 +2752,13 @@ def _refold_parse_tree(parse_tree, *, policy): [ValueTerminal(make_quoted_pairs(p), 'ptext') for p in newparts] + [ValueTerminal('"', 'ptext')]) + if part.token_type == 'comment': + newparts = ( + [ValueTerminal('(', 'ptext')] + + [ValueTerminal(make_parenthesis_pairs(p), 'ptext') + if p.token_type == 'ptext' else p + for p in newparts] + + [ValueTerminal(')', 'ptext')]) if not part.as_ew_allowed: wrap_as_ew_blocked += 1 newparts.append(end_ew_not_allowed) diff --git a/Lib/test/test_email/test__header_value_parser.py b/Lib/test/test_email/test__header_value_parser.py index 9a4e8c7c5a43f5a..62c34f5437253b2 100644 --- a/Lib/test/test_email/test__header_value_parser.py +++ b/Lib/test/test_email/test__header_value_parser.py @@ -2855,6 +2855,29 @@ def test_address_list_with_specials_in_long_quoted_string(self): with self.subTest(to=to): self._test(parser.get_address_list(to)[0], folded, policy=policy) + def test_address_list_with_long_unwrapable_comment(self): + policy = self.policy.clone(max_line_length=40) + cases = [ + # (to, folded) + ('(loremipsumdolorsitametconsecteturadipi)', + '(loremipsumdolorsitametconsecteturadipi)\n'), + ('(loremipsumdolorsitametconsecteturadipi)', + '(loremipsumdolorsitametconsecteturadipi)\n'), + ('(loremipsum dolorsitametconsecteturadipi)', + '(loremipsum dolorsitametconsecteturadipi)\n'), + ('(loremipsum dolorsitametconsecteturadipi)', + '(loremipsum\n dolorsitametconsecteturadipi)\n'), + ('(Escaped \\( \\) chars \\\\ in comments stay escaped)', + '(Escaped \\( \\) chars \\\\ in comments stay\n escaped)\n'), + ('((loremipsum)(loremipsum)(loremipsum)(loremipsum))', + '((loremipsum)(loremipsum)(loremipsum)(loremipsum))\n'), + ('((loremipsum)(loremipsum)(loremipsum) (loremipsum))', + '((loremipsum)(loremipsum)(loremipsum)\n (loremipsum))\n'), + ] + for (to, folded) in cases: + with self.subTest(to=to): + self._test(parser.get_address_list(to)[0], folded, policy=policy) + def test_address_list_with_specials_in_encoded_word(self): # An encoded-word parsed from a structured header must remain # encoded when it contains specials. Regression for gh-121284. diff --git a/Misc/NEWS.d/next/Security/2026-01-16-14-40-31.gh-issue-143935.U2YtKl.rst b/Misc/NEWS.d/next/Security/2026-01-16-14-40-31.gh-issue-143935.U2YtKl.rst new file mode 100644 index 000000000000000..c3d864936884ac0 --- /dev/null +++ b/Misc/NEWS.d/next/Security/2026-01-16-14-40-31.gh-issue-143935.U2YtKl.rst @@ -0,0 +1,6 @@ +Fixed a bug in the folding of comments when flattening an email message +using a modern email policy. Comments consisting of a very long sequence of +non-foldable characters could trigger a forced line wrap that omitted the +required leading space on the continuation line, causing the remainder of +the comment to be interpreted as a new header field. This enabled header +injection with carefully crafted inputs. From 500cd3c554368462abe2079a5fea5b51ffc1657e Mon Sep 17 00:00:00 2001 From: tomcruiseqi Date: Wed, 25 Mar 2026 02:23:45 +0800 Subject: [PATCH 049/129] [3.10] gh-143930: Reject leading dashes in webbrowser URLs (GH-143931) (GH-146359) (cherry picked from commit 82a24a4442312bdcfc4c799885e8b3e00990f02b) Co-authored-by: Seth Michael Larson (cherry picked from commit ad4d5ba32af4d80b0dfa2ba9d8203bfb219e60a5) (cherry picked from commit 58011bdd3931a4715ecea81198e4dba7deab3462) 3.7.17.x: 3.7 has no sys.audit(), so only the _check_url() calls are added. --- Lib/test/test_webbrowser.py | 5 +++++ Lib/webbrowser.py | 14 ++++++++++++++ .../2026-01-16-12-04-49.gh-issue-143930.zYC5x3.rst | 1 + 3 files changed, 20 insertions(+) create mode 100644 Misc/NEWS.d/next/Security/2026-01-16-12-04-49.gh-issue-143930.zYC5x3.rst diff --git a/Lib/test/test_webbrowser.py b/Lib/test/test_webbrowser.py index 519a9432abe012d..f8e9234db82439b 100644 --- a/Lib/test/test_webbrowser.py +++ b/Lib/test/test_webbrowser.py @@ -55,6 +55,11 @@ def test_open(self): options=[], arguments=[URL]) + def test_reject_dash_prefixes(self): + browser = self.browser_class(name=CMD_NAME) + with self.assertRaises(ValueError): + browser.open(f"--key=val {URL}") + class BackgroundBrowserCommandTest(CommandTestMixin, unittest.TestCase): diff --git a/Lib/webbrowser.py b/Lib/webbrowser.py index b04ec7b65ae3b65..72ea9d571be948a 100755 --- a/Lib/webbrowser.py +++ b/Lib/webbrowser.py @@ -154,6 +154,12 @@ def open_new(self, url): def open_new_tab(self, url): return self.open(url, 2) + @staticmethod + def _check_url(url): + """Ensures that the URL is safe to pass to subprocesses as a parameter""" + if url and url.lstrip().startswith("-"): + raise ValueError(f"Invalid URL: {url}") + class GenericBrowser(BaseBrowser): """Class for all browsers started with a command @@ -170,6 +176,7 @@ def __init__(self, name): self.basename = os.path.basename(self.name) def open(self, url, new=0, autoraise=True): + self._check_url(url) cmdline = [self.name] + [arg.replace("%s", url) for arg in self.args] try: @@ -189,6 +196,7 @@ class BackgroundBrowser(GenericBrowser): def open(self, url, new=0, autoraise=True): cmdline = [self.name] + [arg.replace("%s", url) for arg in self.args] + self._check_url(url) try: if sys.platform[:3] == 'win': p = subprocess.Popen(cmdline) @@ -253,6 +261,7 @@ def _invoke(self, args, remote, autoraise): return not p.wait() def open(self, url, new=0, autoraise=True): + self._check_url(url) if new == 0: action = self.remote_action elif new == 1: @@ -353,6 +362,7 @@ class Konqueror(BaseBrowser): """ def open(self, url, new=0, autoraise=True): + self._check_url(url) # XXX Currently I know no way to prevent KFM from opening a new win. if new == 2: action = "newTab" @@ -436,6 +446,7 @@ def _remote(self, action): return 1 def open(self, url, new=0, autoraise=True): + self._check_url(url) if new: ok = self._remote("LOADNEW " + url) else: @@ -593,6 +604,7 @@ def register_standard_browsers(): if sys.platform[:3] == "win": class WindowsDefault(BaseBrowser): def open(self, url, new=0, autoraise=True): + self._check_url(url) try: os.startfile(url) except OSError: @@ -622,6 +634,7 @@ def __init__(self, name): self.name = name def open(self, url, new=0, autoraise=True): + self._check_url(url) assert "'" not in url # hack for local urls if not ':' in url: @@ -659,6 +672,7 @@ def __init__(self, name): self._name = name def open(self, url, new=0, autoraise=True): + self._check_url(url) if self._name == 'default': script = 'open location "%s"' % url.replace('"', '%22') # opens in default browser else: diff --git a/Misc/NEWS.d/next/Security/2026-01-16-12-04-49.gh-issue-143930.zYC5x3.rst b/Misc/NEWS.d/next/Security/2026-01-16-12-04-49.gh-issue-143930.zYC5x3.rst new file mode 100644 index 000000000000000..0f27eae99a0dfd7 --- /dev/null +++ b/Misc/NEWS.d/next/Security/2026-01-16-12-04-49.gh-issue-143930.zYC5x3.rst @@ -0,0 +1 @@ +Reject leading dashes in URLs passed to :func:`webbrowser.open` From 7396392f9e6e1151d49995d20098cdbbc8f68ef2 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Fri, 3 Apr 2026 22:53:24 +0200 Subject: [PATCH 050/129] [3.10] gh-143930: Tweak the exception message and increase test coverage (GH-146476) (GH-148045) (GH-148051) (GH-148053) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit (cherry picked from commit cc023511238ad93ecc8796157c6f9139a2bb2932) (cherry picked from commit 89bfb8e5ed3c7caa241028f1a4eac5f6275a46a4) (cherry picked from commit 3681d47a440865aead912a054d4599087b4270dd) Co-authored-by: Łukasz Langa (cherry picked from commit 591ed890270c5697b013bf637029fb3e6cd2d73e) (cherry picked from commit 6f184879e6d14f576fa189da67dd881e0a0c0cf7) --- Lib/test/test_webbrowser.py | 83 +++++++++++++++++-- Lib/webbrowser.py | 2 +- ...-01-16-12-04-49.gh-issue-143930.zYC5x3.rst | 2 +- 3 files changed, 79 insertions(+), 8 deletions(-) diff --git a/Lib/test/test_webbrowser.py b/Lib/test/test_webbrowser.py index f8e9234db82439b..89d66f473fb0f46 100644 --- a/Lib/test/test_webbrowser.py +++ b/Lib/test/test_webbrowser.py @@ -1,6 +1,7 @@ +import io +import os import webbrowser import unittest -import os import sys import subprocess from unittest import mock @@ -45,6 +46,14 @@ def _test(self, meth, *, args=[URL], kw={}, options, arguments): popen_args.pop(popen_args.index(option)) self.assertEqual(popen_args, arguments) + def test_reject_dash_prefixes(self): + browser = self.browser_class(name=CMD_NAME) + with self.assertRaisesRegex( + ValueError, + r"^Invalid URL \(leading dash disallowed\): '--key=val http.*'$" + ): + browser.open(f"--key=val {URL}") + class GenericBrowserCommandTest(CommandTestMixin, unittest.TestCase): @@ -55,11 +64,6 @@ def test_open(self): options=[], arguments=[URL]) - def test_reject_dash_prefixes(self): - browser = self.browser_class(name=CMD_NAME) - with self.assertRaises(ValueError): - browser.open(f"--key=val {URL}") - class BackgroundBrowserCommandTest(CommandTestMixin, unittest.TestCase): @@ -220,6 +224,73 @@ def test_open_new_tab(self): arguments=['openURL({},new-tab)'.format(URL)]) +class MockPopenPipe: + def __init__(self, cmd, mode): + self.cmd = cmd + self.mode = mode + self.pipe = io.StringIO() + self._closed = False + + def write(self, buf): + self.pipe.write(buf) + + def close(self): + self._closed = True + return None + + +@unittest.skipUnless(sys.platform == "darwin", "macOS specific test") +class MacOSXOSAScriptTest(unittest.TestCase): + def setUp(self): + # Ensure that 'BROWSER' is not set to 'open' or something else. + # See: https://github.com/python/cpython/issues/131254. + ctx = os_helper.EnvironmentVarGuard() + env = ctx.__enter__() + self.addCleanup(ctx.__exit__) + env.unset("BROWSER") + + support.patch(self, os, "popen", self.mock_popen) + self.browser = webbrowser.MacOSXOSAScript("default") + + def mock_popen(self, cmd, mode): + self.popen_pipe = MockPopenPipe(cmd, mode) + return self.popen_pipe + + def test_default(self): + browser = webbrowser.get() + assert isinstance(browser, webbrowser.MacOSXOSAScript) + self.assertEqual(browser._name, "default") + + def test_default_open(self): + url = "https://python.org" + self.browser.open(url) + self.assertTrue(self.popen_pipe._closed) + self.assertEqual(self.popen_pipe.cmd, "osascript") + script = self.popen_pipe.pipe.getvalue() + self.assertEqual(script.strip(), f'open location "{url}"') + + def test_url_quote(self): + self.browser.open('https://python.org/"quote"') + script = self.popen_pipe.pipe.getvalue() + self.assertEqual( + script.strip(), 'open location "https://python.org/%22quote%22"' + ) + + def test_explicit_browser(self): + browser = webbrowser.MacOSXOSAScript("safari") + browser.open("https://python.org") + script = self.popen_pipe.pipe.getvalue() + self.assertIn('tell application "safari"', script) + self.assertIn('open location "https://python.org"', script) + + def test_reject_dash_prefixes(self): + with self.assertRaisesRegex( + ValueError, + r"^Invalid URL \(leading dash disallowed\): '--key=val http.*'$" + ): + self.browser.open(f"--key=val {URL}") + + class BrowserRegistrationTest(unittest.TestCase): def setUp(self): diff --git a/Lib/webbrowser.py b/Lib/webbrowser.py index 72ea9d571be948a..a67e315311b67d2 100755 --- a/Lib/webbrowser.py +++ b/Lib/webbrowser.py @@ -158,7 +158,7 @@ def open_new_tab(self, url): def _check_url(url): """Ensures that the URL is safe to pass to subprocesses as a parameter""" if url and url.lstrip().startswith("-"): - raise ValueError(f"Invalid URL: {url}") + raise ValueError(f"Invalid URL (leading dash disallowed): {url!r}") class GenericBrowser(BaseBrowser): diff --git a/Misc/NEWS.d/next/Security/2026-01-16-12-04-49.gh-issue-143930.zYC5x3.rst b/Misc/NEWS.d/next/Security/2026-01-16-12-04-49.gh-issue-143930.zYC5x3.rst index 0f27eae99a0dfd7..c561023c3c2d7a7 100644 --- a/Misc/NEWS.d/next/Security/2026-01-16-12-04-49.gh-issue-143930.zYC5x3.rst +++ b/Misc/NEWS.d/next/Security/2026-01-16-12-04-49.gh-issue-143930.zYC5x3.rst @@ -1 +1 @@ -Reject leading dashes in URLs passed to :func:`webbrowser.open` +Reject leading dashes in URLs passed to :func:`webbrowser.open`. From 6f4e61a16ee20942618b426fde28e665b360ccf3 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Tue, 7 Apr 2026 05:35:21 +0200 Subject: [PATCH 051/129] [3.10] gh-137586: Open external osascript program with absolute path (GH-137584) (#148177) Co-authored-by: Fionn <1897918+fionn@users.noreply.github.com> Co-authored-by: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> (cherry picked from commit e70ff776fdf8e239060b714ff4edcad3794e1996) (cherry picked from commit 37f05221db73183080397ab27659f7652ea86786) --- Lib/test/test_webbrowser.py | 2 +- Lib/turtledemo/__main__.py | 2 +- Lib/webbrowser.py | 2 +- .../next/macOS/2025-10-17-01-07-03.gh-issue-137586.kVzxvp.rst | 1 + 4 files changed, 4 insertions(+), 3 deletions(-) create mode 100644 Misc/NEWS.d/next/macOS/2025-10-17-01-07-03.gh-issue-137586.kVzxvp.rst diff --git a/Lib/test/test_webbrowser.py b/Lib/test/test_webbrowser.py index 89d66f473fb0f46..a70dd6731fa26e7 100644 --- a/Lib/test/test_webbrowser.py +++ b/Lib/test/test_webbrowser.py @@ -265,7 +265,7 @@ def test_default_open(self): url = "https://python.org" self.browser.open(url) self.assertTrue(self.popen_pipe._closed) - self.assertEqual(self.popen_pipe.cmd, "osascript") + self.assertEqual(self.popen_pipe.cmd, "/usr/bin/osascript") script = self.popen_pipe.pipe.getvalue() self.assertEqual(script.strip(), f'open location "{url}"') diff --git a/Lib/turtledemo/__main__.py b/Lib/turtledemo/__main__.py index 12be5098dad274a..96e7f5004e795cb 100644 --- a/Lib/turtledemo/__main__.py +++ b/Lib/turtledemo/__main__.py @@ -138,7 +138,7 @@ def __init__(self, filename=None): # so that our menu bar appears. subprocess.run( [ - 'osascript', + '/usr/bin/osascript', '-e', 'tell application "System Events"', '-e', 'set frontmost of the first process whose ' 'unix id is {} to true'.format(os.getpid()), diff --git a/Lib/webbrowser.py b/Lib/webbrowser.py index a67e315311b67d2..e3a576ac0cd6ccb 100755 --- a/Lib/webbrowser.py +++ b/Lib/webbrowser.py @@ -683,7 +683,7 @@ def open(self, url, new=0, autoraise=True): end '''%(self._name, url.replace('"', '%22')) - osapipe = os.popen("osascript", "w") + osapipe = os.popen("/usr/bin/osascript", "w") if osapipe is None: return False diff --git a/Misc/NEWS.d/next/macOS/2025-10-17-01-07-03.gh-issue-137586.kVzxvp.rst b/Misc/NEWS.d/next/macOS/2025-10-17-01-07-03.gh-issue-137586.kVzxvp.rst new file mode 100644 index 000000000000000..8e42065392a2de6 --- /dev/null +++ b/Misc/NEWS.d/next/macOS/2025-10-17-01-07-03.gh-issue-137586.kVzxvp.rst @@ -0,0 +1 @@ +Invoke :program:`osascript` with absolute path in :mod:`webbrowser` and :mod:`!turtledemo`. From e440d80b8e23ad72c49d826b7298b8c51d8b4003 Mon Sep 17 00:00:00 2001 From: Stan Ulbrych Date: Wed, 8 Apr 2026 11:27:42 +0100 Subject: [PATCH 052/129] [3.10] gh-145986: Avoid unbound C recursion in `conv_content_model` in `pyexpat.c` (CVE 2026-4224) (GH-145987) (#146002) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * [3.10] gh-145986: Avoid unbound C recursion in `conv_content_model` in `pyexpat.c` (CVE 2026-4224) (GH-145987) Fix C stack overflow (CVE-2026-4224) when an Expat parser with a registered `ElementDeclHandler` parses inline DTD containing deeply nested content model. --------- (cherry picked from commit eb0e8be3a7e11b87d198a2c3af1ed0eccf532768) (cherry picked from commit e5caf45faac74b0ed869e3336420cffd3510ce6e) Co-authored-by: Stan Ulbrych <89152624+StanFromIreland@users.noreply.github.com> Co-authored-by: Bénédikt Tran <10796600+picnixz@users.noreply.github.com> * Update Misc/NEWS.d/next/Security/2026-03-14-17-31-39.gh-issue-145986.ifSSr8.rst --------- Co-authored-by: Bénédikt Tran <10796600+picnixz@users.noreply.github.com> (cherry picked from commit af856a7177326ac25d9f66cc6dd28b554d914fee) (cherry picked from commit 446494ffaa16324436e558024bdcea3ef725dd63) 3.7.17.x: test.support.infinite_recursion() added for the test. --- Lib/test/support/__init__.py | 10 ++++++++++ Lib/test/test_pyexpat.py | 19 +++++++++++++++++++ ...-03-14-17-31-39.gh-issue-145986.ifSSr8.rst | 4 ++++ Modules/pyexpat.c | 8 +++++++- 4 files changed, 40 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Security/2026-03-14-17-31-39.gh-issue-145986.ifSSr8.rst diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index 3a19d1d6f8d822f..017f6a60c5a98da 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -3046,6 +3046,16 @@ def adjust_int_max_str_digits(max_digits): sys.set_int_max_str_digits(current) +@contextlib.contextmanager +def infinite_recursion(max_depth=75): + original_depth = sys.getrecursionlimit() + try: + sys.setrecursionlimit(max_depth) + yield + finally: + sys.setrecursionlimit(original_depth) + + def control_characters_c0(): """Returns a list of C0 control characters as strings. C0 control characters defined as the byte range 0x00-0x1F, and 0x7F. diff --git a/Lib/test/test_pyexpat.py b/Lib/test/test_pyexpat.py index 413c237679d99ff..bbc2f9b4213e034 100644 --- a/Lib/test/test_pyexpat.py +++ b/Lib/test/test_pyexpat.py @@ -11,6 +11,7 @@ from xml.parsers import expat from xml.parsers.expat import errors +from test.support import infinite_recursion from test.support import sortdict @@ -642,6 +643,24 @@ def test_change_size_2(self): parser.Parse(xml2, 1) self.assertEqual(self.n, 4) +class ElementDeclHandlerTest(unittest.TestCase): + def test_deeply_nested_content_model(self): + # This should raise a RecursionError and not crash. + # See https://github.com/python/cpython/issues/145986. + N = 500_000 + data = ( + b'\n]>\n\n' + ) + + parser = expat.ParserCreate() + parser.ElementDeclHandler = lambda _1, _2: None + with infinite_recursion(): + with self.assertRaises(RecursionError): + parser.Parse(data) + + class MalformedInputTest(unittest.TestCase): def test1(self): xml = b"\0\r\n" diff --git a/Misc/NEWS.d/next/Security/2026-03-14-17-31-39.gh-issue-145986.ifSSr8.rst b/Misc/NEWS.d/next/Security/2026-03-14-17-31-39.gh-issue-145986.ifSSr8.rst new file mode 100644 index 000000000000000..cb9dbadb72d976d --- /dev/null +++ b/Misc/NEWS.d/next/Security/2026-03-14-17-31-39.gh-issue-145986.ifSSr8.rst @@ -0,0 +1,4 @@ +:mod:`xml.parsers.expat`: Fixed a crash caused by unbounded C recursion when +converting deeply nested XML content models with +:meth:`~xml.parsers.expat.xmlparser.ElementDeclHandler`. +This addresses `CVE-2026-4224 `_. diff --git a/Modules/pyexpat.c b/Modules/pyexpat.c index b8386758da5fd02..00449fd03463762 100644 --- a/Modules/pyexpat.c +++ b/Modules/pyexpat.c @@ -534,6 +534,10 @@ static PyObject * conv_content_model(XML_Content * const model, PyObject *(*conv_string)(const XML_Char *)) { + if (Py_EnterRecursiveCall(" in conv_content_model")) { + return NULL; + } + PyObject *result = NULL; PyObject *children = PyTuple_New(model->numchildren); int i; @@ -545,7 +549,7 @@ conv_content_model(XML_Content * const model, conv_string); if (child == NULL) { Py_XDECREF(children); - return NULL; + goto done; } PyTuple_SET_ITEM(children, i, child); } @@ -553,6 +557,8 @@ conv_content_model(XML_Content * const model, model->type, model->quant, conv_string,model->name, children); } +done: + Py_LeaveRecursiveCall(); return result; } From 45cb5733805248e3c696b6ad92ebc9476dce07d5 Mon Sep 17 00:00:00 2001 From: Stan Ulbrych Date: Mon, 13 Apr 2026 22:41:53 +0100 Subject: [PATCH 053/129] [3.10] gh-148169: Fix webbrowser `%action` substitution bypass of dash-prefix check (GH-148170) (#148521) (cherry picked from commit d22922c8a7958353689dc4763dd72da2dea03fff) (cherry picked from commit c5767a72838a8dda9d6dc5d3558075b055c56bca) (cherry picked from commit beb8e2769108cc911bfda4e6bf0c890236a15c56) 3.7.17.x: no sys.audit() in 3.7. --- Lib/test/test_webbrowser.py | 8 ++++++++ Lib/webbrowser.py | 5 +++-- .../2026-03-31-09-15-51.gh-issue-148169.EZJzz2.rst | 2 ++ 3 files changed, 13 insertions(+), 2 deletions(-) create mode 100644 Misc/NEWS.d/next/Security/2026-03-31-09-15-51.gh-issue-148169.EZJzz2.rst diff --git a/Lib/test/test_webbrowser.py b/Lib/test/test_webbrowser.py index a70dd6731fa26e7..123b474318a36ae 100644 --- a/Lib/test/test_webbrowser.py +++ b/Lib/test/test_webbrowser.py @@ -99,6 +99,14 @@ def test_open_new_tab(self): options=[], arguments=[URL]) + def test_reject_action_dash_prefixes(self): + browser = self.browser_class(name=CMD_NAME) + with self.assertRaises(ValueError): + browser.open('%action--incognito') + # new=1: action is "--new-window", so "%action" itself expands to + # a dash-prefixed flag even with no dash in the original URL. + with self.assertRaises(ValueError): + browser.open('%action', new=1) class MozillaCommandTest(CommandTestMixin, unittest.TestCase): diff --git a/Lib/webbrowser.py b/Lib/webbrowser.py index e3a576ac0cd6ccb..b180644e4b59ad2 100755 --- a/Lib/webbrowser.py +++ b/Lib/webbrowser.py @@ -261,7 +261,6 @@ def _invoke(self, args, remote, autoraise): return not p.wait() def open(self, url, new=0, autoraise=True): - self._check_url(url) if new == 0: action = self.remote_action elif new == 1: @@ -275,7 +274,9 @@ def open(self, url, new=0, autoraise=True): raise Error("Bad 'new' parameter to open(); " + "expected 0, 1, or 2, got %s" % new) - args = [arg.replace("%s", url).replace("%action", action) + self._check_url(url.replace("%action", action)) + + args = [arg.replace("%action", action).replace("%s", url) for arg in self.remote_args] args = [arg for arg in args if arg] success = self._invoke(args, True, autoraise) diff --git a/Misc/NEWS.d/next/Security/2026-03-31-09-15-51.gh-issue-148169.EZJzz2.rst b/Misc/NEWS.d/next/Security/2026-03-31-09-15-51.gh-issue-148169.EZJzz2.rst new file mode 100644 index 000000000000000..45cdeebe1b6d64a --- /dev/null +++ b/Misc/NEWS.d/next/Security/2026-03-31-09-15-51.gh-issue-148169.EZJzz2.rst @@ -0,0 +1,2 @@ +A bypass in :mod:`webbrowser` allowed URLs prefixed with ``%action`` to pass +the dash-prefix safety check. From 9616e2634c3934ad0d49bd731589cdb1c518682a Mon Sep 17 00:00:00 2001 From: Stan Ulbrych Date: Mon, 13 Apr 2026 22:42:24 +0100 Subject: [PATCH 054/129] [3.10] gh-148395: Fix a possible UAF in `{LZMA,BZ2}Decompressor` (GH-148396) (#148505) Fix dangling input pointer after `MemoryError` in _lzma/_bz2/_ZlibDecompressor.decompress (cherry picked from commit 8fc66aef6d7b3ae58f43f5c66f9366cc8cbbfcd2) (cherry picked from commit 47128e64f98c3a20271138a98c2922bea2a3ee0e) (cherry picked from commit 2dc27d509dd9d5c53d3dad8262e0a7d85ead4922) --- .../Security/2026-04-10-16-28-21.gh-issue-148395.kfzm0G.rst | 5 +++++ Modules/_bz2module.c | 1 + Modules/_lzmamodule.c | 1 + 3 files changed, 7 insertions(+) create mode 100644 Misc/NEWS.d/next/Security/2026-04-10-16-28-21.gh-issue-148395.kfzm0G.rst diff --git a/Misc/NEWS.d/next/Security/2026-04-10-16-28-21.gh-issue-148395.kfzm0G.rst b/Misc/NEWS.d/next/Security/2026-04-10-16-28-21.gh-issue-148395.kfzm0G.rst new file mode 100644 index 000000000000000..b095329bd913b03 --- /dev/null +++ b/Misc/NEWS.d/next/Security/2026-04-10-16-28-21.gh-issue-148395.kfzm0G.rst @@ -0,0 +1,5 @@ +Fix a dangling input pointer in :class:`lzma.LZMADecompressor`, +:class:`bz2.BZ2Decompressor` +when memory allocation fails with :exc:`MemoryError`, which could let a +subsequent :meth:`!decompress` call read or write through a stale pointer to +the already-released caller buffer. diff --git a/Modules/_bz2module.c b/Modules/_bz2module.c index f0d9588fe55d6f9..448a7125bab4779 100644 --- a/Modules/_bz2module.c +++ b/Modules/_bz2module.c @@ -570,6 +570,7 @@ decompress(BZ2Decompressor *d, char *data, size_t len, Py_ssize_t max_length) return result; error: + bzs->next_in = NULL; Py_XDECREF(result); return NULL; } diff --git a/Modules/_lzmamodule.c b/Modules/_lzmamodule.c index ec370897495083c..ed4ff203947aa2b 100644 --- a/Modules/_lzmamodule.c +++ b/Modules/_lzmamodule.c @@ -1052,6 +1052,7 @@ decompress(Decompressor *d, uint8_t *data, size_t len, Py_ssize_t max_length) return result; error: + lzs->next_in = NULL; Py_XDECREF(result); return NULL; } From 82d7aacde69b544098d9c2fc023169e617145d7c Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Thu, 30 Apr 2026 23:18:52 +0200 Subject: [PATCH 055/129] [3.10] gh-141707: Skip TarInfo DIRTYPE normalization during GNU long name handling (#145816) gh-141707: Skip TarInfo DIRTYPE normalization during GNU long name handling (cherry picked from commit 42d754e34c06e57ad6b8e7f92f32af679912d8ab) Co-authored-by: Seth Michael Larson Co-authored-by: Eashwar Ranganathan (cherry picked from commit 72dde1016493c52abe857fc4a7bf6c40138b4114) (cherry picked from commit c9236adf364314ae1ea6bd6e3bdae869df78c414) 3.7.17.x: 3.7's GNUReadTest and PaxReadTest get the format attribute the test uses. --- Lib/tarfile.py | 29 ++++++++++++++++--- Lib/test/test_tarfile.py | 21 ++++++++++++++ Misc/ACKS | 1 + ...-11-18-06-35-53.gh-issue-141707.DBmQIy.rst | 2 ++ 4 files changed, 49 insertions(+), 4 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2025-11-18-06-35-53.gh-issue-141707.DBmQIy.rst diff --git a/Lib/tarfile.py b/Lib/tarfile.py index 9301aebcedb0b01..e8bf0335001f985 100755 --- a/Lib/tarfile.py +++ b/Lib/tarfile.py @@ -1225,6 +1225,20 @@ def _create_pax_generic_header(cls, pax_headers, type, encoding): @classmethod def frombuf(cls, buf, encoding, errors): """Construct a TarInfo object from a 512 byte bytes object. + + To support the old v7 tar format AREGTYPE headers are + transformed to DIRTYPE headers if their name ends in '/'. + """ + return cls._frombuf(buf, encoding, errors) + + @classmethod + def _frombuf(cls, buf, encoding, errors, *, dircheck=True): + """Construct a TarInfo object from a 512 byte bytes object. + + If ``dircheck`` is set to ``True`` then ``AREGTYPE`` headers will + be normalized to ``DIRTYPE`` if the name ends in a trailing slash. + ``dircheck`` must be set to ``False`` if this function is called + on a follow-up header such as ``GNUTYPE_LONGNAME``. """ if len(buf) == 0: raise EmptyHeaderError("empty header") @@ -1255,7 +1269,7 @@ def frombuf(cls, buf, encoding, errors): # Old V7 tar format represents a directory as a regular # file with a trailing slash. - if obj.type == AREGTYPE and obj.name.endswith("/"): + if dircheck and obj.type == AREGTYPE and obj.name.endswith("/"): obj.type = DIRTYPE # The old GNU sparse format occupies some of the unused @@ -1290,8 +1304,15 @@ def fromtarfile(cls, tarfile): """Return the next TarInfo object from TarFile object tarfile. """ + return cls._fromtarfile(tarfile) + + @classmethod + def _fromtarfile(cls, tarfile, *, dircheck=True): + """ + See dircheck documentation in _frombuf(). + """ buf = tarfile.fileobj.read(BLOCKSIZE) - obj = cls.frombuf(buf, tarfile.encoding, tarfile.errors) + obj = cls._frombuf(buf, tarfile.encoding, tarfile.errors, dircheck=dircheck) obj.offset = tarfile.fileobj.tell() - BLOCKSIZE return obj._proc_member(tarfile) @@ -1344,7 +1365,7 @@ def _proc_gnulong(self, tarfile): # Fetch the next header and process it. try: - next = self.fromtarfile(tarfile) + next = self._fromtarfile(tarfile, dircheck=False) except HeaderError: raise SubsequentHeaderError("missing or bad subsequent header") @@ -1475,7 +1496,7 @@ def _proc_pax(self, tarfile): # Fetch the next header. try: - next = self.fromtarfile(tarfile) + next = self._fromtarfile(tarfile, dircheck=False) except HeaderError: raise SubsequentHeaderError("missing or bad subsequent header") diff --git a/Lib/test/test_tarfile.py b/Lib/test/test_tarfile.py index 8bba3b67031b7ec..15e3e9cca048675 100644 --- a/Lib/test/test_tarfile.py +++ b/Lib/test/test_tarfile.py @@ -946,10 +946,30 @@ def test_header_offset(self): self.assertEqual(tarinfo.type, self.longnametype) + def test_longname_file_not_directory(self): + # Test reading a longname file and ensure it is not handled as a directory + # Issue #141707 + buf = io.BytesIO() + with tarfile.open(mode='w', fileobj=buf, format=self.format) as tar: + ti = tarfile.TarInfo() + ti.type = tarfile.AREGTYPE + ti.name = ('a' * 99) + '/' + ('b' * 3) + tar.addfile(ti) + + expected = {t.name: t.type for t in tar.getmembers()} + + buf.seek(0) + with tarfile.open(mode='r', fileobj=buf) as tar: + actual = {t.name: t.type for t in tar.getmembers()} + + self.assertEqual(expected, actual) + + class GNUReadTest(LongnameTest, ReadTest, unittest.TestCase): subdir = "gnu" longnametype = tarfile.GNUTYPE_LONGNAME + format = tarfile.GNU_FORMAT # Since 3.2 tarfile is supposed to accurately restore sparse members and # produce files with holes. This is what we actually want to test here. @@ -1009,6 +1029,7 @@ class PaxReadTest(LongnameTest, ReadTest, unittest.TestCase): subdir = "pax" longnametype = tarfile.XHDTYPE + format = tarfile.PAX_FORMAT def test_pax_global_headers(self): tar = tarfile.open(tarname, encoding="iso8859-1") diff --git a/Misc/ACKS b/Misc/ACKS index bd62c958c5d2de9..32b3f191f305012 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -1322,6 +1322,7 @@ Dhushyanth Ramasamy Ashwin Ramaswami Jeff Ramnani Bayard Randel +Eashwar Ranganathan Varpu Rantala Brodie Rao Rémi Rampin diff --git a/Misc/NEWS.d/next/Library/2025-11-18-06-35-53.gh-issue-141707.DBmQIy.rst b/Misc/NEWS.d/next/Library/2025-11-18-06-35-53.gh-issue-141707.DBmQIy.rst new file mode 100644 index 000000000000000..1f5b8ed90b8a904 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-11-18-06-35-53.gh-issue-141707.DBmQIy.rst @@ -0,0 +1,2 @@ +Don't change :class:`tarfile.TarInfo` type from ``AREGTYPE`` to ``DIRTYPE`` when parsing +GNU long name or link headers. From 1f51b59ac246eb397851de7a3ebfbbd72b738056 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sun, 17 May 2026 20:33:20 +0200 Subject: [PATCH 056/129] [3.10] gh-144370: Disallow usage of control characters in status in wsgiref.handlers for security (#144371) (#145673) (cherry picked from commit 10cf7e2bf76171f7c3b050a9acd762320e00486b) (cherry picked from commit f1a360329f2876145aa535dbbfc66c188400878c) --- Lib/test/test_wsgiref.py | 19 +++++++++++++++++++ Lib/wsgiref/handlers.py | 4 +++- Misc/ACKS | 1 + ...-01-31-21-56-54.gh-issue-144370.fp9m8t.rst | 2 ++ 4 files changed, 25 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Security/2026-01-31-21-56-54.gh-issue-144370.fp9m8t.rst diff --git a/Lib/test/test_wsgiref.py b/Lib/test/test_wsgiref.py index edacb1e7160ba8d..398c73acfb46fb1 100644 --- a/Lib/test/test_wsgiref.py +++ b/Lib/test/test_wsgiref.py @@ -870,6 +870,25 @@ def write(self, b): self.assertIsNotNone(h.status) self.assertIsNotNone(h.environ) + def testRaisesControlCharacters(self): + for c0 in control_characters_c0(): + with self.subTest(c0): + base = BaseHandler() + with self.assertRaises(ValueError): + base.start_response(c0, [('x', 'y')]) + + base = BaseHandler() + with self.assertRaises(ValueError): + base.start_response('200 OK', [(c0, 'y')]) + + # HTAB (\x09) is allowed in header values, but not in names. + base = BaseHandler() + if c0 != "\t": + with self.assertRaises(ValueError): + base.start_response('200 OK', [('x', c0)]) + else: + base.start_response('200 OK', [('x', c0)]) + if __name__ == "__main__": unittest.main() diff --git a/Lib/wsgiref/handlers.py b/Lib/wsgiref/handlers.py index eee3f948c723060..85827d60aed79db 100644 --- a/Lib/wsgiref/handlers.py +++ b/Lib/wsgiref/handlers.py @@ -1,7 +1,7 @@ """Base classes for server/gateway implementations""" from .util import FileWrapper, guess_scheme, is_hop_by_hop -from .headers import Headers +from .headers import Headers, _name_disallowed_re import sys, os, time @@ -238,6 +238,8 @@ def start_response(self, status, headers,exc_info=None): self.status = status self.headers = self.headers_class(headers) status = self._convert_string_type(status, "Status") + if _name_disallowed_re.search(status): + raise ValueError("Control characters are not allowed in status") assert len(status)>=4,"Status must be at least 4 characters" assert status[:3].isdigit(), "Status message must begin w/3-digit code" assert status[3]==" ", "Status message must have a space after code" diff --git a/Misc/ACKS b/Misc/ACKS index 32b3f191f305012..eea91613615eb54 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -911,6 +911,7 @@ Wolfgang Langner Detlef Lannert Rémi Lapeyre Soren Larsen +Seth Michael Larson Amos Latteier Piers Lauder Ben Laurie diff --git a/Misc/NEWS.d/next/Security/2026-01-31-21-56-54.gh-issue-144370.fp9m8t.rst b/Misc/NEWS.d/next/Security/2026-01-31-21-56-54.gh-issue-144370.fp9m8t.rst new file mode 100644 index 000000000000000..2d13a0611322c52 --- /dev/null +++ b/Misc/NEWS.d/next/Security/2026-01-31-21-56-54.gh-issue-144370.fp9m8t.rst @@ -0,0 +1,2 @@ +Disallow usage of control characters in status in :mod:`wsgiref.handlers` to prevent HTTP header injections. +Patch by Benedikt Johannes. From 27e0b7f191a6fd9d7e130a245f957723b7a0b16f Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Sat, 27 Jun 2026 19:32:34 +0200 Subject: [PATCH 057/129] [3.10] gh-151981: Make tarfile._Stream.seek break at EOF (GH-151982) (#151996) gh-151981: Make tarfile._Stream.seek break at EOF (GH-151982) (cherry picked from commit f50bf13566189c8d0ce5a814f33eff3d89951896) Co-authored-by: Petr Viktorin Co-authored-by: Stan Ulbrych (cherry picked from commit eb63c0f94dfcbea7fda8eab6213818e134d67192) (cherry picked from commit c37e6143672aa9645cdb83bf363e7f0916f5dde7) --- Lib/tarfile.py | 4 +++- Lib/test/test_tarfile.py | 16 ++++++++++++++++ ...026-06-23-13-28-16.gh-issue-151981.xBHEcU.rst | 2 ++ 3 files changed, 21 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Security/2026-06-23-13-28-16.gh-issue-151981.xBHEcU.rst diff --git a/Lib/tarfile.py b/Lib/tarfile.py index e8bf0335001f985..4de42c83fae97cf 100755 --- a/Lib/tarfile.py +++ b/Lib/tarfile.py @@ -518,7 +518,9 @@ def seek(self, pos=0): if pos - self.pos >= 0: blocks, remainder = divmod(pos - self.pos, self.bufsize) for i in range(blocks): - self.read(self.bufsize) + data = self.read(self.bufsize) + if not data: + break self.read(remainder) else: raise StreamError("seeking backwards is not allowed") diff --git a/Lib/test/test_tarfile.py b/Lib/test/test_tarfile.py index 15e3e9cca048675..0b186d9b2998093 100644 --- a/Lib/test/test_tarfile.py +++ b/Lib/test/test_tarfile.py @@ -4054,6 +4054,22 @@ def valueerror_filter(tarinfo, path): with self.check_context(arc.open(errorlevel='boo!'), filtererror_filter): self.expect_exception(TypeError) # errorlevel is not int + @support.subTests('format', [tarfile.GNU_FORMAT, tarfile.PAX_FORMAT]) + def test_getmembers_big_size(self, format): + # gh-151981: A loop in seek() for streaming files tried to read the + # declared number of blocks even at EOF + tinfo = tarfile.TarInfo("huge-file") + tinfo.size = 1 << 64 + bio = io.BytesIO() + # Write header without data + bio.write(tinfo.tobuf(format)) + + # Reset & try to get contents + bio.seek(0) + with tarfile.open(fileobj=bio, mode="r|") as tar: + with self.assertRaises(tarfile.ReadError): + tar.getmembers() + class OffsetValidationTests(unittest.TestCase): tarname = tmpname diff --git a/Misc/NEWS.d/next/Security/2026-06-23-13-28-16.gh-issue-151981.xBHEcU.rst b/Misc/NEWS.d/next/Security/2026-06-23-13-28-16.gh-issue-151981.xBHEcU.rst new file mode 100644 index 000000000000000..2123ab8e081b1d9 --- /dev/null +++ b/Misc/NEWS.d/next/Security/2026-06-23-13-28-16.gh-issue-151981.xBHEcU.rst @@ -0,0 +1,2 @@ +In :mod:`tarfile`, seeking a stream now stops when end of the stream is +reached. From e0a2872dc57a259c5aa7336ded03b000f93df6eb Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Sat, 27 Jun 2026 19:57:57 +0200 Subject: [PATCH 058/129] [3.10] gh-87451: Apply CVE-2021-4189 PASV fix to ftplib.ftpcp() (GH-149648) (#149797) gh-87451: Apply CVE-2021-4189 PASV fix to ftplib.ftpcp() (GH-149648) ftpcp() called parse227() directly and passed the source server's self-reported PASV IPv4 address to the target server's PORT command, bypassing the CVE-2021-4189 fix that was applied only to FTP.makepasv(). A malicious source FTP server could use this to redirect the target server's data connection to an arbitrary host:port (SSRF). ftpcp() now uses the source server's actual peer address, honoring the existing trust_server_pasv_ipv4_address opt-out, the same as makepasv(). Thanks to Qi Ding at Aurascape AI for the report. (GHSA-w8c5-q2xf-gf7c) (cherry picked from commit eac4fe3b2c77693790a5ef7dfab127c1fee81bf9) Co-authored-by: Gregory P. Smith <68491+gpshead@users.noreply.github.com> (cherry picked from commit 2bbcf3fb7a420a05605576c0f9468d4675381b5f) (cherry picked from commit 7ff917f9e170193c7e8c9a8bdf986bfeb364416c) --- Lib/ftplib.py | 11 +++++- Lib/test/test_ftplib.py | 36 ++++++++++++++++++- ...6-05-10-18-05-32.gh-issue-87451.XkKB6M.rst | 6 ++++ 3 files changed, 51 insertions(+), 2 deletions(-) create mode 100644 Misc/NEWS.d/next/Security/2026-05-10-18-05-32.gh-issue-87451.XkKB6M.rst diff --git a/Lib/ftplib.py b/Lib/ftplib.py index 83ed0aa9c140e7e..4e16a509e1c57cd 100644 --- a/Lib/ftplib.py +++ b/Lib/ftplib.py @@ -920,7 +920,16 @@ def ftpcp(source, sourcename, target, targetname = '', type = 'I'): type = 'TYPE ' + type source.voidcmd(type) target.voidcmd(type) - sourcehost, sourceport = parse227(source.sendcmd('PASV')) + # Don't trust the IPv4 address the source server advertises in its PASV + # reply: a malicious source could otherwise point the target's data + # connection at an arbitrary host (SSRF). A caller that needs the old + # behavior can set trust_server_pasv_ipv4_address on the source FTP + # object. See FTP.makepasv(), which applies the same rule. + untrusted_host, sourceport = parse227(source.sendcmd('PASV')) + if source.trust_server_pasv_ipv4_address: + sourcehost = untrusted_host + else: + sourcehost = source.sock.getpeername()[0] target.sendport(sourcehost, sourceport) # RFC 959: the user must "listen" [...] BEFORE sending the # transfer request. diff --git a/Lib/test/test_ftplib.py b/Lib/test/test_ftplib.py index 73a6250a5e4ae90..c047c7137b8808d 100644 --- a/Lib/test/test_ftplib.py +++ b/Lib/test/test_ftplib.py @@ -17,7 +17,7 @@ except ImportError: ssl = None -from unittest import TestCase, skipUnless +from unittest import mock, TestCase, skipUnless from test import support from test.support import HOST, HOSTv6 @@ -1096,6 +1096,40 @@ def testTimeoutDirectAccess(self): ftp.close() +class TestFtpcpSecurity(TestCase): + """ftpcp() must not trust the host a source server advertises in PASV. + + A malicious source server can otherwise redirect the target server's + data connection to an arbitrary host:port (SSRF), so ftpcp() uses the + source server's actual peer address instead, the same as FTP.makepasv(). + """ + + def _make_pair(self, *, advertised_host, real_host, trust=False): + source = mock.Mock(spec=ftplib.FTP) + source.trust_server_pasv_ipv4_address = trust + source.sock.getpeername.return_value = (real_host, 21) + # PASV replies give the host as comma-separated octets, not dotted. + advertised = advertised_host.replace('.', ',') + source.sendcmd.side_effect = lambda cmd: ( + f'227 Entering Passive Mode ({advertised},1,2).' + if cmd == 'PASV' else '150 ok') + target = mock.Mock(spec=ftplib.FTP) + target.sendcmd.return_value = '150 ok' + return source, target + + def test_ftpcp_ignores_untrusted_pasv_host(self): + source, target = self._make_pair(advertised_host='10.0.0.5', + real_host='198.51.100.7') + ftplib.ftpcp(source, 'a', target, 'b') + target.sendport.assert_called_once_with('198.51.100.7', 258) + + def test_ftpcp_trust_server_pasv_ipv4_address(self): + source, target = self._make_pair(advertised_host='10.0.0.5', + real_host='198.51.100.7', trust=True) + ftplib.ftpcp(source, 'a', target, 'b') + target.sendport.assert_called_once_with('10.0.0.5', 258) + + class MiscTestCase(TestCase): def test__all__(self): blacklist = {'MSG_OOB', 'FTP_PORT', 'MAXLINE', 'CRLF', 'B_CRLF', diff --git a/Misc/NEWS.d/next/Security/2026-05-10-18-05-32.gh-issue-87451.XkKB6M.rst b/Misc/NEWS.d/next/Security/2026-05-10-18-05-32.gh-issue-87451.XkKB6M.rst new file mode 100644 index 000000000000000..21a79c3e0e7db74 --- /dev/null +++ b/Misc/NEWS.d/next/Security/2026-05-10-18-05-32.gh-issue-87451.XkKB6M.rst @@ -0,0 +1,6 @@ +The :mod:`ftplib` module's undocumented ``ftpcp`` function no longer trusts +the IPv4 address value returned from the source server in response to the +``PASV`` command by default, completing the fix for CVE-2021-4189. As with +:class:`ftplib.FTP`, the former behavior can be re-enabled by setting the +``trust_server_pasv_ipv4_address`` attribute on the source :class:`ftplib.FTP` +instance to ``True``. Thanks to Qi Deng at Aurascape AI for the report. From 56dbc23ea20d69cf727e81464327a5e92b9a903f Mon Sep 17 00:00:00 2001 From: Seth Larson Date: Sat, 27 Jun 2026 12:59:15 -0500 Subject: [PATCH 059/129] =?UTF-8?q?[3.10]=20gh-146211:=20Reject=20CR/LF=20?= =?UTF-8?q?in=20HTTP=20tunnel=20request=20headers=20(GH-146=E2=80=A6=20(#1?= =?UTF-8?q?48350)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit [3.10] gh-146211: Reject CR/LF in HTTP tunnel request headers (GH-146212) (cherry picked from commit 05ed7ce7ae9e17c23a04085b2539fe6d6d3cef69) Co-authored-by: Illia Volochii (cherry picked from commit 58703ec1bdd1eb075e8b01a0c427683ce594dd3e) (cherry picked from commit 612c005e7343c68ad35aa630de7984f4effe7c1a) --- Lib/http/client.py | 27 +++++++---- Lib/test/test_httplib.py | 45 +++++++++++++++++++ ...-03-20-09-29-42.gh-issue-146211.PQVbs7.rst | 2 + 3 files changed, 66 insertions(+), 8 deletions(-) create mode 100644 Misc/NEWS.d/next/Security/2026-03-20-09-29-42.gh-issue-146211.PQVbs7.rst diff --git a/Lib/http/client.py b/Lib/http/client.py index 640019153456778..df0884417c2f7d0 100644 --- a/Lib/http/client.py +++ b/Lib/http/client.py @@ -928,15 +928,26 @@ def set_debuglevel(self, level): self.debuglevel = level def _tunnel(self): - connect_str = "CONNECT %s:%d HTTP/1.0\r\n" % (self._tunnel_host, - self._tunnel_port) - connect_bytes = connect_str.encode("ascii") - self.send(connect_bytes) + if _contains_disallowed_url_pchar_re.search(self._tunnel_host): + raise ValueError('Tunnel host can\'t contain control characters %r' + % (self._tunnel_host,)) + connect = b"CONNECT %s:%d HTTP/1.0\r\n" % ( + self._tunnel_host.encode("ascii"), self._tunnel_port) + headers = [connect] for header, value in self._tunnel_headers.items(): - header_str = "%s: %s\r\n" % (header, value) - header_bytes = header_str.encode("latin-1") - self.send(header_bytes) - self.send(b'\r\n') + header_bytes = header.encode("latin-1") + value_bytes = value.encode("latin-1") + if not _is_legal_header_name(header_bytes): + raise ValueError('Invalid header name %r' % (header_bytes,)) + if _is_illegal_header_value(value_bytes): + raise ValueError('Invalid header value %r' % (value_bytes,)) + headers.append(b"%s: %s\r\n" % (header_bytes, value_bytes)) + headers.append(b"\r\n") + # Making a single send() call instead of one per line encourages + # the host OS to use a more optimal packet size instead of + # potentially emitting a series of small packets. + self.send(b"".join(headers)) + del headers response = self.response_class(self.sock, method=self._method) (version, code, message) = response._read_status() diff --git a/Lib/test/test_httplib.py b/Lib/test/test_httplib.py index 6f24bca4d7b7d37..59bfed812b58129 100644 --- a/Lib/test/test_httplib.py +++ b/Lib/test/test_httplib.py @@ -345,6 +345,51 @@ def test_invalid_headers(self): with self.assertRaisesRegex(ValueError, 'Invalid header'): conn.putheader(name, value) + def test_invalid_tunnel_headers(self): + cases = ( + ('Invalid\r\nName', 'ValidValue'), + ('Invalid\rName', 'ValidValue'), + ('Invalid\nName', 'ValidValue'), + ('\r\nInvalidName', 'ValidValue'), + ('\rInvalidName', 'ValidValue'), + ('\nInvalidName', 'ValidValue'), + (' InvalidName', 'ValidValue'), + ('\tInvalidName', 'ValidValue'), + ('Invalid:Name', 'ValidValue'), + (':InvalidName', 'ValidValue'), + ('ValidName', 'Invalid\r\nValue'), + ('ValidName', 'Invalid\rValue'), + ('ValidName', 'Invalid\nValue'), + ('ValidName', 'InvalidValue\r\n'), + ('ValidName', 'InvalidValue\r'), + ('ValidName', 'InvalidValue\n'), + ) + for name, value in cases: + with self.subTest((name, value)): + conn = client.HTTPConnection('example.com') + conn.set_tunnel('tunnel', headers={ + name: value + }) + conn.sock = FakeSocket('') + with self.assertRaisesRegex(ValueError, 'Invalid header'): + conn._tunnel() # Called in .connect() + + def test_invalid_tunnel_host(self): + cases = ( + 'invalid\r.host', + '\ninvalid.host', + 'invalid.host\r\n', + 'invalid.host\x00', + 'invalid host', + ) + for tunnel_host in cases: + with self.subTest(tunnel_host): + conn = client.HTTPConnection('example.com') + conn.set_tunnel(tunnel_host) + conn.sock = FakeSocket('') + with self.assertRaisesRegex(ValueError, 'Tunnel host can\'t contain control characters'): + conn._tunnel() # Called in .connect() + def test_headers_debuglevel(self): body = ( b'HTTP/1.1 200 OK\r\n' diff --git a/Misc/NEWS.d/next/Security/2026-03-20-09-29-42.gh-issue-146211.PQVbs7.rst b/Misc/NEWS.d/next/Security/2026-03-20-09-29-42.gh-issue-146211.PQVbs7.rst new file mode 100644 index 000000000000000..4993633b8ebebb2 --- /dev/null +++ b/Misc/NEWS.d/next/Security/2026-03-20-09-29-42.gh-issue-146211.PQVbs7.rst @@ -0,0 +1,2 @@ +Reject CR/LF characters in tunnel request headers for the +HTTPConnection.set_tunnel() method. From a279f931caa2b5177307156cb28d6fbc40c5b90a Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sat, 27 Jun 2026 20:00:01 +0200 Subject: [PATCH 060/129] [3.10] gh-145599, CVE 2026-3644: Reject control characters in `http.cookies.Morsel.update()` (#145600) (#146027) * gh-145599, CVE 2026-3644: Reject control characters in `http.cookies.Morsel.update()` (#145600) Reject control characters in `http.cookies.Morsel.update()` and `http.cookies.BaseCookie.js_output`. Co-authored-by: Victor Stinner Co-authored-by: Victor Stinner (cherry picked from commit 57e88c1cf95e1481b94ae57abe1010469d47a6b4) * Update Misc/NEWS.d/next/Security/2026-03-06-17-03-38.gh-issue-145599.kchwZV.rst Co-authored-by: Stan Ulbrych <89152624+StanFromIreland@users.noreply.github.com> --------- Co-authored-by: Stan Ulbrych <89152624+StanFromIreland@users.noreply.github.com> (cherry picked from commit 556aa098e738b127c714866f819b4abe2f7593d8) (cherry picked from commit 1d91959c8fb08f6d1588bfcf45829038174d2850) --- Lib/http/cookies.py | 24 ++++++++++-- Lib/test/test_http_cookies.py | 39 +++++++++++++++++++ ...-03-06-17-03-38.gh-issue-145599.kchwZV.rst | 4 ++ 3 files changed, 63 insertions(+), 4 deletions(-) create mode 100644 Misc/NEWS.d/next/Security/2026-03-06-17-03-38.gh-issue-145599.kchwZV.rst diff --git a/Lib/http/cookies.py b/Lib/http/cookies.py index 31ade9f2ed94c3e..c5658beb06719ec 100644 --- a/Lib/http/cookies.py +++ b/Lib/http/cookies.py @@ -334,9 +334,16 @@ def update(self, values): key = key.lower() if key not in self._reserved: raise CookieError("Invalid attribute %r" % (key,)) + if _has_control_character(key, val): + raise CookieError("Control characters are not allowed in " + f"cookies {key!r} {val!r}") data[key] = val dict.update(self, data) + def __ior__(self, values): + self.update(values) + return self + def isReservedKey(self, K): return K.lower() in self._reserved @@ -362,9 +369,15 @@ def __getstate__(self): } def __setstate__(self, state): - self._key = state['key'] - self._value = state['value'] - self._coded_value = state['coded_value'] + key = state['key'] + value = state['value'] + coded_value = state['coded_value'] + if _has_control_character(key, value, coded_value): + raise CookieError("Control characters are not allowed in cookies " + f"{key!r} {value!r} {coded_value!r}") + self._key = key + self._value = value + self._coded_value = coded_value def output(self, attrs=None, header="Set-Cookie:"): return "%s %s" % (header, self.OutputString(attrs)) @@ -376,13 +389,16 @@ def __repr__(self): def js_output(self, attrs=None): # Print javascript + output_string = self.OutputString(attrs) + if _has_control_character(output_string): + raise CookieError("Control characters are not allowed in cookies") return """ - """ % (self.OutputString(attrs).replace('"', r'\"')) + """ % (output_string.replace('"', r'\"')) def OutputString(self, attrs=None): # Build up our result diff --git a/Lib/test/test_http_cookies.py b/Lib/test/test_http_cookies.py index 659d547e84e8f8d..08107a0655c4989 100644 --- a/Lib/test/test_http_cookies.py +++ b/Lib/test/test_http_cookies.py @@ -514,6 +514,14 @@ def test_control_characters(self): with self.assertRaises(cookies.CookieError): morsel["path"] = c0 + # .__setstate__() + with self.assertRaises(cookies.CookieError): + morsel.__setstate__({'key': c0, 'value': 'val', 'coded_value': 'coded'}) + with self.assertRaises(cookies.CookieError): + morsel.__setstate__({'key': 'key', 'value': c0, 'coded_value': 'coded'}) + with self.assertRaises(cookies.CookieError): + morsel.__setstate__({'key': 'key', 'value': 'val', 'coded_value': c0}) + # .setdefault() with self.assertRaises(cookies.CookieError): morsel.setdefault("path", c0) @@ -528,6 +536,18 @@ def test_control_characters(self): with self.assertRaises(cookies.CookieError): morsel.set("path", "val", c0) + # .update() + with self.assertRaises(cookies.CookieError): + morsel.update({"path": c0}) + with self.assertRaises(cookies.CookieError): + morsel.update({c0: "val"}) + + # .__ior__() + with self.assertRaises(cookies.CookieError): + morsel |= {"path": c0} + with self.assertRaises(cookies.CookieError): + morsel |= {c0: "val"} + def test_control_characters_output(self): # Tests that even if the internals of Morsel are modified # that a call to .output() has control character safeguards. @@ -548,6 +568,25 @@ def test_control_characters_output(self): with self.assertRaises(cookies.CookieError): cookie.output() + # Tests that .js_output() also has control character safeguards. + for c0 in support.control_characters_c0(): + morsel = cookies.Morsel() + morsel.set("key", "value", "coded-value") + morsel._key = c0 # Override private variable. + cookie = cookies.SimpleCookie() + cookie["cookie"] = morsel + with self.assertRaises(cookies.CookieError): + cookie.js_output() + + morsel = cookies.Morsel() + morsel.set("key", "value", "coded-value") + morsel._coded_value = c0 # Override private variable. + cookie = cookies.SimpleCookie() + cookie["cookie"] = morsel + with self.assertRaises(cookies.CookieError): + cookie.js_output() + + def test_main(): run_unittest(CookieTests, MorselTests) run_doctest(cookies) diff --git a/Misc/NEWS.d/next/Security/2026-03-06-17-03-38.gh-issue-145599.kchwZV.rst b/Misc/NEWS.d/next/Security/2026-03-06-17-03-38.gh-issue-145599.kchwZV.rst new file mode 100644 index 000000000000000..fc2e503779a2546 --- /dev/null +++ b/Misc/NEWS.d/next/Security/2026-03-06-17-03-38.gh-issue-145599.kchwZV.rst @@ -0,0 +1,4 @@ +Reject control characters in :class:`http.cookies.Morsel` +:meth:`~http.cookies.Morsel.update` and +:meth:`~http.cookies.BaseCookie.js_output`. +This addresses `CVE-2026-3644 `_. From 7be289bbd5cac0d5e7d0a1da0626be5aa80900f1 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Sun, 28 Jun 2026 22:50:36 +0200 Subject: [PATCH 061/129] [3.10] gh-150743: Limit trailer lines and interim responses read by http.client (GH-150749) (#152529) (cherry picked from commit 1790e584142b5db070b74bc64777ad14e26608c2) (cherry picked from commit 8e95088569ff2a3db88a53bebb841c119ccfbc6a) --- Lib/http/client.py | 21 ++++++- Lib/test/test_httplib.py | 58 +++++++++++++++++++ ...05-30-00-00-00.gh-issue-150743.httpdos.rst | 6 ++ 3 files changed, 84 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Security/2026-05-30-00-00-00.gh-issue-150743.httpdos.rst diff --git a/Lib/http/client.py b/Lib/http/client.py index df0884417c2f7d0..753821bd03ddd1b 100644 --- a/Lib/http/client.py +++ b/Lib/http/client.py @@ -112,6 +112,13 @@ _MAXLINE = 65536 _MAXHEADERS = 100 +# maximal number of interim (1xx) responses tolerated before the final +# response. Real servers send at most a few; without a bound, a server +# streaming "100 Continue" responses would hang getresponse() forever. +# A socket timeout cannot detect that as data keeps arriving within every +# timeout window. +_MAXINTERIMRESPONSES = 100 + # Data larger than this will be read in chunks, to prevent extreme # overallocation. _MIN_READ_BUF_SIZE = 1 << 20 @@ -320,7 +327,7 @@ def begin(self): return # read until we get a non-100 response - while True: + for _ in range(_MAXINTERIMRESPONSES): version, status, reason = self._read_status() if status != CONTINUE: break @@ -329,6 +336,9 @@ def begin(self): if self.debuglevel > 0: print("headers:", skipped_headers) del skipped_headers + else: + raise HTTPException( + f"got more than {_MAXINTERIMRESPONSES} interim responses") self.code = self.status = status self.reason = reason.strip() @@ -541,6 +551,7 @@ def _read_next_chunk_size(self): def _read_and_discard_trailer(self): # read and discard trailer up to the CRLF terminator ### note: we shouldn't have any trailers! + trailers_read = 0 while True: line = self.fp.readline(_MAXLINE + 1) if len(line) > _MAXLINE: @@ -551,6 +562,14 @@ def _read_and_discard_trailer(self): break if line in (b'\r\n', b'\n', b''): break + # Bound the trailer count just as response headers are bounded. + # A server streaming trailer lines forever would otherwise hang + # the client; a socket timeout cannot detect that as data keeps + # arriving within every timeout window. + trailers_read += 1 + if trailers_read > _MAXHEADERS: + raise HTTPException( + f"got more than {_MAXHEADERS} trailers") def _get_chunk_left(self): # return self.chunk_left, reading a new chunk if necessary. diff --git a/Lib/test/test_httplib.py b/Lib/test/test_httplib.py index 59bfed812b58129..0136125274cf4e6 100644 --- a/Lib/test/test_httplib.py +++ b/Lib/test/test_httplib.py @@ -1056,6 +1056,35 @@ def test_overflowing_header_limit_after_100(self): self.assertIn('got more than ', str(cm.exception)) self.assertIn('headers', str(cm.exception)) + def test_too_many_interim_responses(self): + # A server streaming "100 Continue" responses forever must not + # hang getresponse(). + body = ( + 'HTTP/1.1 100 Continue\r\n\r\n' + * (client._MAXINTERIMRESPONSES + 1) + ) + resp = client.HTTPResponse(FakeSocket(body)) + with self.assertRaises(client.HTTPException) as cm: + resp.begin() + self.assertIn('got more than ', str(cm.exception)) + self.assertIn('interim responses', str(cm.exception)) + + def test_multiple_interim_responses(self): + # A reasonable number of interim responses before the final + # response is skipped as before. + body = ( + 'HTTP/1.1 100 Continue\r\n\r\n' * 3 + + 'HTTP/1.1 200 OK\r\n' + 'Content-Length: 5\r\n' + '\r\n' + 'hello' + ) + resp = client.HTTPResponse(FakeSocket(body), method="GET") + resp.begin() + self.assertEqual(resp.status, 200) + self.assertEqual(resp.read(), b'hello') + resp.close() + def test_overflowing_chunked_line(self): body = ( 'HTTP/1.1 200 OK\r\n' @@ -1127,6 +1156,35 @@ def test_chunked_trailers(self): self.assertEqual(sock.file.read(), b"") #we read to the end resp.close() + def test_chunked_too_many_trailers(self): + """A response streaming endless trailer lines must raise, not hang""" + too_many_trailers = "".join( + f"X-Trailer{i}: {i}\r\n" for i in range(client._MAXHEADERS + 1) + ) + # An unbounded read() reaches the trailers via the final 0 chunk. + sock = FakeSocket( + chunked_start + last_chunk + too_many_trailers + chunked_end) + resp = client.HTTPResponse(sock, method="GET") + resp.begin() + with self.assertRaisesRegex( + client.HTTPException, + f"got more than {client._MAXHEADERS} trailers", + ): + resp.read() + resp.close() + + # A bounded read(amt) larger than the body hits the same limit. + sock = FakeSocket( + chunked_start + last_chunk + too_many_trailers + chunked_end) + resp = client.HTTPResponse(sock, method="GET") + resp.begin() + with self.assertRaisesRegex( + client.HTTPException, + f"got more than {client._MAXHEADERS} trailers", + ): + resp.read(len(chunked_expected) + 1) + resp.close() + def test_chunked_sync(self): """Check that we don't read past the end of the chunked-encoding stream""" expected = chunked_expected diff --git a/Misc/NEWS.d/next/Security/2026-05-30-00-00-00.gh-issue-150743.httpdos.rst b/Misc/NEWS.d/next/Security/2026-05-30-00-00-00.gh-issue-150743.httpdos.rst new file mode 100644 index 000000000000000..4cc4e4b092da71c --- /dev/null +++ b/Misc/NEWS.d/next/Security/2026-05-30-00-00-00.gh-issue-150743.httpdos.rst @@ -0,0 +1,6 @@ +:mod:`http.client` now limits the number of chunked-response trailer lines +it will read to 100, and the number of interim (1xx) responses it will +skip to 100. A malicious or broken server could previously stream trailer +lines or ``100 Continue`` responses forever, hanging the client even when +a socket timeout was in use. +Reported by ``@YLChen-007`` via GHSA-w4q2-g22w-6fr4. From 7e474226e05905065c72fbbaa17275526857ee40 Mon Sep 17 00:00:00 2001 From: Stan Ulbrych Date: Sat, 4 Jul 2026 15:15:47 +0200 Subject: [PATCH 062/129] [3.10] gh-150599: Prevent bz2 decompressor reuse after errors (#150600) (#151133) * [3.12] gh-150599: Prevent bz2 decompressor reuse after errors (#150600) (#151054) (cherry picked from commit 5755d0f083949ff3c5bf3a37e673e24e306b036e) * We can just make it a plain assignment, 1a9cdaf63af7014dd7bd852b4d8a8c0ab98387ab was 3.15+ anyway. (cherry picked from commit 938ec030e90c5e53f1faac6fab1643f14e4f4a79) (cherry picked from commit fd1215febf068280e71f470272b84be738dee13c) --- Lib/test/test_bz2.py | 15 +++++++++++++++ ...6-05-30-09-36-20.gh-issue-150599.nlHqU-.rst | 3 +++ Modules/_bz2module.c | 18 +++++++++++++++--- 3 files changed, 33 insertions(+), 3 deletions(-) create mode 100644 Misc/NEWS.d/next/Security/2026-05-30-09-36-20.gh-issue-150599.nlHqU-.rst diff --git a/Lib/test/test_bz2.py b/Lib/test/test_bz2.py index 003497f28b16244..1780822b65ec202 100644 --- a/Lib/test/test_bz2.py +++ b/Lib/test/test_bz2.py @@ -817,6 +817,21 @@ def test_failure(self): # Previously, a second call could crash due to internal inconsistency self.assertRaises(Exception, bzd.decompress, self.BAD_DATA * 30) + def test_decompress_after_data_error(self): + data = bytes.fromhex( + "425a6839314159265359000000000000007fffff000000000000000000000000" + "00000000000000000000000000000000000000e0370000000000000000000000" + "000000000000000000000000000000000000000000000000000083f3" + ) + bzd = BZ2Decompressor() + with self.assertRaisesRegex(OSError, "Invalid data stream"): + bzd.decompress(data) + # Previously, a second call could crash due to internal inconsistency + self.assertFalse(bzd.needs_input) + self.assertFalse(bzd.eof) + with self.assertRaisesRegex(ValueError, "previous error"): + bzd.decompress(b'\x00' * 18) + @support.refcount_test def test_refleaks_in___init__(self): gettotalrefcount = support.get_attribute(sys, 'gettotalrefcount') diff --git a/Misc/NEWS.d/next/Security/2026-05-30-09-36-20.gh-issue-150599.nlHqU-.rst b/Misc/NEWS.d/next/Security/2026-05-30-09-36-20.gh-issue-150599.nlHqU-.rst new file mode 100644 index 000000000000000..a37d86cf423f820 --- /dev/null +++ b/Misc/NEWS.d/next/Security/2026-05-30-09-36-20.gh-issue-150599.nlHqU-.rst @@ -0,0 +1,3 @@ +Fix a possible stack buffer overflow in :mod:`bz2` when a +:class:`bz2.BZ2Decompressor` is reused after a decompression error. +The decompressor now becomes unusable after libbz2 reports an error. diff --git a/Modules/_bz2module.c b/Modules/_bz2module.c index 448a7125bab4779..ca3ccb753c38d22 100644 --- a/Modules/_bz2module.c +++ b/Modules/_bz2module.c @@ -40,6 +40,7 @@ typedef struct { typedef struct { PyObject_HEAD bz_stream bzs; + int bzerror; char eof; /* T_BOOL expects a char */ PyObject *unused_data; char needs_input; @@ -436,8 +437,11 @@ decompress_buf(BZ2Decompressor *d, Py_ssize_t max_length) data_size = bzs->next_out - PyBytes_AS_STRING(result); d->bzs_avail_in_real += bzs->avail_in; Py_END_ALLOW_THREADS - if (catch_bz2_error(bzret)) + if (catch_bz2_error(bzret)) { + d->bzerror = bzret; + d->needs_input = 0; goto error; + } if (bzret == BZ_STREAM_END) { d->eof = 1; break; @@ -605,10 +609,17 @@ _bz2_BZ2Decompressor_decompress_impl(BZ2Decompressor *self, Py_buffer *data, PyObject *result = NULL; ACQUIRE_LOCK(self); - if (self->eof) + if (self->eof) { PyErr_SetString(PyExc_EOFError, "End of stream already reached"); - else + } + else if (self->bzerror) { + // Re-entering BZ2_bzDecompress() after an error can write out of bounds. + PyErr_SetString(PyExc_ValueError, + "Decompressor is unusable after a previous error"); + } + else { result = decompress(self, data->buf, data->len, max_length); + } RELEASE_LOCK(self); return result; } @@ -645,6 +656,7 @@ _bz2_BZ2Decompressor___init___impl(BZ2Decompressor *self) } self->lock = lock; + self->bzerror = 0; self->needs_input = 1; self->bzs_avail_in_real = 0; self->input_buffer = NULL; From 276076588d1c2ef7918a51d882956c60cbd1af20 Mon Sep 17 00:00:00 2001 From: Stan Ulbrych Date: Mon, 6 Jul 2026 21:42:04 +0200 Subject: [PATCH 063/129] [3.10] gh-149486: tarfile.data_filter: validate written link target (GH-149487) (#153180) (cherry picked from commit c063191cb7f9170f9565e305f8aa2b79ab2bf609) (cherry picked from commit b6f6db1e861ad6de98389983ba521f7a28396f0f) --- Lib/tarfile.py | 16 +- Lib/test/test_tarfile.py | 147 ++++++++++++++---- ...-05-03-21-00-00.gh-issue-149486.tarflt.rst | 5 + 3 files changed, 129 insertions(+), 39 deletions(-) create mode 100644 Misc/NEWS.d/next/Security/2026-05-03-21-00-00.gh-issue-149486.tarflt.rst diff --git a/Lib/tarfile.py b/Lib/tarfile.py index 4de42c83fae97cf..595d2cf05fbd81c 100755 --- a/Lib/tarfile.py +++ b/Lib/tarfile.py @@ -825,16 +825,22 @@ def _get_filtered_attrs(member, dest_path, for_data=True): if member.islnk() or member.issym(): if os.path.isabs(member.linkname): raise AbsoluteLinkError(member) + # A link member that resolves to the destination directory itself + # would replace it with a (sym)link, redirecting the destination + # for all subsequent members. + if target_path == dest_path: + raise OutsideDestinationError(member, target_path) normalized = os.path.normpath(member.linkname) if normalized != member.linkname: new_attrs['linkname'] = normalized if member.issym(): - target_path = os.path.join(dest_path, - os.path.dirname(name), - member.linkname) + # The symlink is created at `name` with trailing separators + # stripped, so its target is relative to the directory + # containing that path. + link_dir = os.path.dirname(name.rstrip('/' + os.sep)) + target_path = os.path.join(dest_path, link_dir, normalized) else: - target_path = os.path.join(dest_path, - member.linkname) + target_path = os.path.join(dest_path, normalized) target_path = os.path.realpath(target_path, strict=os.path.ALLOW_MISSING) if os.path.commonpath([target_path, dest_path]) != dest_path: diff --git a/Lib/test/test_tarfile.py b/Lib/test/test_tarfile.py index 0b186d9b2998093..3365cec44228356 100644 --- a/Lib/test/test_tarfile.py +++ b/Lib/test/test_tarfile.py @@ -3132,6 +3132,39 @@ class TestExtractionFilters(unittest.TestCase): # The destination for the extraction, within `outerdir` destdir = outerdir / 'dest' + @classmethod + def setUpClass(cls): + # Posix and Windows have different pathname resolution: + # either symlink or a '..' component resolve first. + # Let's see which we are on. + if support.can_symlink(): + testpath = os.path.join(TEMPDIR, 'resolution_test') + os.mkdir(testpath) + + # testpath/current links to `.` which is all of: + # - `testpath` + # - `testpath/current` + # - `testpath/current/current` + # - etc. + os.symlink('.', os.path.join(testpath, 'current')) + + # we'll test where `testpath/current/../file` ends up + with open(os.path.join(testpath, 'current', '..', 'file'), 'w'): + pass + + if os.path.exists(os.path.join(testpath, 'file')): + # Windows collapses 'current\..' to '.' first, leaving + # 'testpath\file' + cls.dotdot_resolves_early = True + elif os.path.exists(os.path.join(testpath, '..', 'file')): + # Posix resolves 'current' to '.' first, leaving + # 'testpath/../file' + cls.dotdot_resolves_early = False + else: + raise AssertionError('Could not determine link resolution') + else: + cls.dotdot_resolves_early = False + @contextmanager def check_context(self, tar, filter, *, check_flag=True, ignored_trees=()): """Extracts `tar` to `self.destdir` and allows checking the result @@ -3323,10 +3356,19 @@ def test_parent_symlink(self): + "which is outside the destination") with self.check_context(arc.open(), 'data'): - self.expect_exception( - tarfile.LinkOutsideDestinationError, - """'parent' would link to ['"].*outerdir['"], """ - + "which is outside the destination") + if self.dotdot_resolves_early: + # 'current/../..' normalises to '..', which is rejected. + self.expect_exception( + tarfile.LinkOutsideDestinationError, + """'parent' would link to ['"].*outerdir['"], """ + + "which is outside the destination") + else: + # 'current/..' normalises to '.'; the rewritten link is + # created and 'parent/evil' lands harmlessly inside the + # destination. + self.expect_file('current', symlink_to='.') + self.expect_file('parent', symlink_to='.') + self.expect_file('evil') else: # No symlink support. The symlinks are ignored. @@ -3413,35 +3455,6 @@ def test_parent_symlink2(self): # Test interplaying symlinks # Inspired by 'dirsymlink2b' in jwilk/traversal-archives - # Posix and Windows have different pathname resolution: - # either symlink or a '..' component resolve first. - # Let's see which we are on. - if support.can_symlink(): - testpath = os.path.join(TEMPDIR, 'resolution_test') - os.mkdir(testpath) - - # testpath/current links to `.` which is all of: - # - `testpath` - # - `testpath/current` - # - `testpath/current/current` - # - etc. - os.symlink('.', os.path.join(testpath, 'current')) - - # we'll test where `testpath/current/../file` ends up - with open(os.path.join(testpath, 'current', '..', 'file'), 'w'): - pass - - if os.path.exists(os.path.join(testpath, 'file')): - # Windows collapses 'current\..' to '.' first, leaving - # 'testpath\file' - dotdot_resolves_early = True - elif os.path.exists(os.path.join(testpath, '..', 'file')): - # Posix resolves 'current' to '.' first, leaving - # 'testpath/../file' - dotdot_resolves_early = False - else: - raise AssertionError('Could not determine link resolution') - with ArchiveMaker() as arc: # `current` links to `.` which is both the destination directory @@ -3477,7 +3490,7 @@ def test_parent_symlink2(self): with self.check_context(arc.open(), 'data'): if support.can_symlink(): - if dotdot_resolves_early: + if self.dotdot_resolves_early: # Fail when extracting a file outside destination self.expect_exception( tarfile.OutsideDestinationError, @@ -3594,6 +3607,72 @@ def test_sly_relative2(self): + """['"].*moo['"], which is outside the """ + "destination") + @support.skip_unless_symlink + def test_normpath_realpath_mismatch(self): + # The link-target check must validate the value that will actually + # be written to disk (the normalised linkname), not the original. + # Here 'a' is a symlink to a deep nonexistent path, so realpath() + # of 'a/../../...' stays inside the destination while normpath() + # collapses 'a/..' lexically and escapes. + depth = len(self.destdir.parts) + 5 + deep = '/'.join(f'p{i}' for i in range(depth)) + sneaky = 'a/' + '../' * depth + 'flag' + for kind in 'symlink_to', 'hardlink_to': + with self.subTest(kind): + with ArchiveMaker() as arc: + arc.add('a', symlink_to=deep) + arc.add('escape', **{kind: sneaky}) + with self.check_context(arc.open(), 'data'): + self.expect_exception( + tarfile.LinkOutsideDestinationError) + + @support.skip_unless_symlink + def test_symlink_trailing_slash(self): + # A trailing slash on a symlink member's name must not cause the + # link target to be resolved relative to the wrong directory. + with ArchiveMaker() as arc: + t = tarfile.TarInfo('x/') + t.type = tarfile.SYMTYPE + t.linkname = '..' + arc.tar_w.addfile(t) + arc.add('x/escaped', content='hi') + + with self.check_context(arc.open(), 'data'): + self.expect_exception(tarfile.LinkOutsideDestinationError) + + @support.skip_unless_symlink + def test_link_at_destination(self): + # A link member whose name resolves to the destination directory + # itself must be rejected: otherwise the destination is replaced + # by a symlink and later members can be redirected through it. + for name in '', '.', './': + with ArchiveMaker() as arc: + t = tarfile.TarInfo(name) + t.type = tarfile.SYMTYPE + t.linkname = '.' + arc.tar_w.addfile(t) + + with self.check_context(arc.open(), 'data'): + self.expect_exception(tarfile.OutsideDestinationError) + + @support.skip_unless_symlink + def test_empty_name_symlink_chain(self): + # Regression test for a chain of empty-named symlinks that + # incrementally redirects the destination outwards. + with ArchiveMaker() as arc: + for name, target in [('', ''), ('a/', '..'), + ('', 'dummy'), ('', 'a'), + ('b/', '..'), + ('', 'dummy'), ('', 'a/b')]: + t = tarfile.TarInfo(name) + t.type = tarfile.SYMTYPE + t.linkname = target + arc.tar_w.addfile(t) + arc.add('escaped', content='hi') + + with self.check_context(arc.open(), 'data'): + self.expect_exception(tarfile.FilterError) + def test_deep_symlink(self): # Test that symlinks and hardlinks inside a directory # point to the correct file (`target` of size 3). diff --git a/Misc/NEWS.d/next/Security/2026-05-03-21-00-00.gh-issue-149486.tarflt.rst b/Misc/NEWS.d/next/Security/2026-05-03-21-00-00.gh-issue-149486.tarflt.rst new file mode 100644 index 000000000000000..7c69edb683cf80a --- /dev/null +++ b/Misc/NEWS.d/next/Security/2026-05-03-21-00-00.gh-issue-149486.tarflt.rst @@ -0,0 +1,5 @@ +:func:`tarfile.data_filter` now validates link targets using the same +normalised value that is written to disk, strips trailing separators from +the member name when resolving a symlink's directory, and rejects link +members that would replace the destination directory itself. This closes +several path-traversal bypasses of the ``data`` extraction filter. From 61479c3ccecad48bcce4acbe8e9f7a059ccff808 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Sat, 11 Jul 2026 19:45:05 +0200 Subject: [PATCH 064/129] [3.10] gh-151987: Pass filter_function to TarFile._extract_one() during .extract() (GH-151988) (#152613) * gh-151987: Pass filter_function to TarFile._extract_one() during .extract() (GH-151988) (cherry picked from commit 7ccdbaba2c54250a70d7f25632152df7655a5e0a) Co-authored-by: Petr Viktorin Co-authored-by: Seth Michael Larson * Remove newer test helpers 3.11+ has more granular platform support, mostly for WASI. Use the 3.10 idioms instead: - @symlink_test is a no-op on non-WASI platforms - can_chmod is approximated by "not win32" elsewhere in test_tarfile --------- Co-authored-by: Petr Viktorin Co-authored-by: Seth Michael Larson (cherry picked from commit cf23b9153181062150d061468b6d24af33fe214f) (cherry picked from commit 328e33f71a24f9324489f4823b34173567c526b7) 3.7.17.x: the tests' parenthesized with-statements (3.10+ syntax) are rewritten with backslash continuations, and use test.support.temp_dir. --- Lib/tarfile.py | 3 +- Lib/test/test_tarfile.py | 86 +++++++++++++++++++ ...-06-23-14-19-30.gh-issue-151987.8mNIMf.rst | 2 + 3 files changed, 90 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Security/2026-06-23-14-19-30.gh-issue-151987.8mNIMf.rst diff --git a/Lib/tarfile.py b/Lib/tarfile.py index 595d2cf05fbd81c..16736ebe3737f18 100755 --- a/Lib/tarfile.py +++ b/Lib/tarfile.py @@ -2353,7 +2353,8 @@ def extract(self, member, path="", set_attrs=True, *, numeric_owner=False, tarinfo, unfiltered = self._get_extract_tarinfo( member, filter_function, path) if tarinfo is not None: - self._extract_one(tarinfo, path, set_attrs, numeric_owner) + self._extract_one(tarinfo, path, set_attrs, numeric_owner, + filter_function=filter_function) def _get_extract_tarinfo(self, member, filter_function, path): """Get (filtered, unfiltered) TarInfos from *member* diff --git a/Lib/test/test_tarfile.py b/Lib/test/test_tarfile.py index 3365cec44228356..897a518158c7e2f 100644 --- a/Lib/test/test_tarfile.py +++ b/Lib/test/test_tarfile.py @@ -3825,6 +3825,92 @@ def test_chmod_outside_dir(self): st_mode = cc.outerdir.stat().st_mode self.assertNotEqual(st_mode & 0o777, 0o777) + @unittest.skipUnless(hasattr(os, 'chown'), "missing os.chown") + @unittest.skipUnless(hasattr(os, 'lchown'), "missing os.lchown") + @unittest.skipUnless(hasattr(os, 'geteuid'), "missing os.geteuid") + @support.subTests('link_type', (tarfile.SYMTYPE, tarfile.LNKTYPE)) + def test_chown_links_on_extract(self, link_type): + with ArchiveMaker() as arc: + arc.add("test.txt", + uid=1337, gid=1337, uname="", gname="", mode='-rwxr-xr-x') + arc.add("link", + type=link_type, + linkname='test.txt', + uid=1337, gid=1337, uname="", gname="", mode='-rwxr-xr-x') + + with support.temp_dir() as tmpdir, \ + arc.open() as tar, \ + unittest.mock.patch("os.chown") as mock_chown, \ + unittest.mock.patch("os.lchown") as mock_lchown, \ + unittest.mock.patch("os.geteuid") as mock_geteuid: + # Set UID to 0 so chown() is attempted. + mock_geteuid.return_value = 0 + tar.extract("link", path=tmpdir, filter='data') + extract_path = os.path.join(tmpdir, "link") + + if link_type == tarfile.SYMTYPE: + mock_chown.assert_not_called() + mock_lchown.assert_called_once_with(extract_path, -1, -1) + else: + mock_chown.assert_has_calls([ + unittest.mock.call(extract_path, -1, -1), + unittest.mock.call(extract_path, -1, -1) + ]) + mock_lchown.assert_not_called() + + @unittest.skipUnless(hasattr(os, 'chown'), "missing os.chown") + @unittest.skipUnless(hasattr(os, 'lchown'), "missing os.lchown") + @unittest.skipUnless(hasattr(os, 'geteuid'), "missing os.geteuid") + @support.subTests('link_type', (tarfile.SYMTYPE, tarfile.LNKTYPE)) + def test_chown_links_on_extractall(self, link_type): + with ArchiveMaker() as arc: + arc.add("test.txt", + uid=1337, gid=1337, uname="", gname="", mode='-rwxr-xr-x') + arc.add("link", + type=link_type, + linkname='test.txt', + uid=1337, gid=1337, uname="", gname="", mode='-rwxr-xr-x') + + with support.temp_dir() as tmpdir, \ + arc.open() as tar, \ + unittest.mock.patch("os.chown") as mock_chown, \ + unittest.mock.patch("os.lchown") as mock_lchown, \ + unittest.mock.patch("os.geteuid") as mock_geteuid: + # Set UID to 0 so chown() is attempted. + mock_geteuid.return_value = 0 + tar.extractall(path=tmpdir, filter='data') + extract_link_path = os.path.join(tmpdir, "link") + extract_file_path = os.path.join(tmpdir, "test.txt") + + if link_type == tarfile.SYMTYPE: + mock_chown.assert_called_once_with(extract_file_path, -1, -1) + mock_lchown.assert_called_once_with(extract_link_path, -1, -1) + else: + mock_chown.assert_has_calls([ + unittest.mock.call(extract_file_path, -1, -1), + unittest.mock.call(extract_link_path, -1, -1) + ]) + mock_lchown.assert_not_called() + + def test_extract_filters_target(self): + # Test that when extract() falls back to extracting (rather than + # linking) a hardlink target, it filters the target. + with ArchiveMaker() as arc: + arc.add("target") + arc.add("link", hardlink_to="target") + def testing_filter(member, path): + if member.name == 'target': + # target: set read-only + return member.replace(mode=stat.S_IRUSR) + # link: don't overwrite the mode + return member.replace(mode=None) + tempdir = pathlib.Path(TEMPDIR) / 'extract' + with support.temp_dir(tempdir), arc.open() as tar: + tar.extract("link", path=tempdir, filter=testing_filter) + path = tempdir / 'link' + if sys.platform != 'win32': + self.assertFalse(path.stat().st_mode & stat.S_IWUSR) + def test_link_fallback_normalizes(self): # Make sure hardlink fallbacks work for non-normalized paths for all # filters diff --git a/Misc/NEWS.d/next/Security/2026-06-23-14-19-30.gh-issue-151987.8mNIMf.rst b/Misc/NEWS.d/next/Security/2026-06-23-14-19-30.gh-issue-151987.8mNIMf.rst new file mode 100644 index 000000000000000..9eea7b32c4d2b4f --- /dev/null +++ b/Misc/NEWS.d/next/Security/2026-06-23-14-19-30.gh-issue-151987.8mNIMf.rst @@ -0,0 +1,2 @@ +The :meth:`tarfile.TarFile.extract` method now applies the given filter when +it extracts a link target from the archive as a fallback. From f2216068c692d32e6cc85b1921133dace01a5a2a Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Mon, 10 Aug 2026 09:11:29 +0200 Subject: [PATCH 065/129] [3.10] gh-153438: Update NuGet download URL (GH-153482) (GH-153515) (#153610) Co-authored-by: Steve Dower Co-authored-by: Harjoth Khara (cherry picked from commit bd3939e002a01a97a82018f90309b7cbc506a4b3) (cherry picked from commit cf5f991190255f9cf5740805546f846e970b0587) --- Doc/using/windows.rst | 6 +++--- .../Build/2026-07-09-22-45-00.gh-issue-153438.Qr7N2p.rst | 2 ++ PCbuild/find_python.bat | 2 +- Tools/msi/get_externals.bat | 2 +- 4 files changed, 7 insertions(+), 5 deletions(-) create mode 100644 Misc/NEWS.d/next/Build/2026-07-09-22-45-00.gh-issue-153438.Qr7N2p.rst diff --git a/Doc/using/windows.rst b/Doc/using/windows.rst index 819d3e830749882..03ddb6b255f873a 100644 --- a/Doc/using/windows.rst +++ b/Doc/using/windows.rst @@ -360,9 +360,9 @@ on using nuget. What follows is a summary that is sufficient for Python developers. The ``nuget.exe`` command line tool may be downloaded directly from -``https://aka.ms/nugetclidl``, for example, using curl or PowerShell. With the -tool, the latest version of Python for 64-bit or 32-bit machines is installed -using:: +``https://dist.nuget.org/win-x86-commandline/latest/nuget.exe``, for example, +using curl or PowerShell. With the tool, the latest version of Python for +64-bit or 32-bit machines is installed using:: nuget.exe install python -ExcludeVersion -OutputDirectory . nuget.exe install pythonx86 -ExcludeVersion -OutputDirectory . diff --git a/Misc/NEWS.d/next/Build/2026-07-09-22-45-00.gh-issue-153438.Qr7N2p.rst b/Misc/NEWS.d/next/Build/2026-07-09-22-45-00.gh-issue-153438.Qr7N2p.rst new file mode 100644 index 000000000000000..edab8e6ba7b2595 --- /dev/null +++ b/Misc/NEWS.d/next/Build/2026-07-09-22-45-00.gh-issue-153438.Qr7N2p.rst @@ -0,0 +1,2 @@ +Update Windows build and installer tooling and documentation to use the +current download URL for ``nuget.exe``. diff --git a/PCbuild/find_python.bat b/PCbuild/find_python.bat index 40937a67c33ac7a..1064f106f8da1cb 100644 --- a/PCbuild/find_python.bat +++ b/PCbuild/find_python.bat @@ -45,7 +45,7 @@ @set _Py_HOST_PYTHON=%HOST_PYTHON% @if "%_Py_HOST_PYTHON%"=="" set _Py_HOST_PYTHON=py @if "%_Py_NUGET%"=="" (set _Py_NUGET=%_Py_EXTERNALS_DIR%\nuget.exe) -@if "%_Py_NUGET_URL%"=="" (set _Py_NUGET_URL=https://aka.ms/nugetclidl) +@if "%_Py_NUGET_URL%"=="" (set _Py_NUGET_URL=https://dist.nuget.org/win-x86-commandline/latest/nuget.exe) @if NOT exist "%_Py_NUGET%" ( @echo Downloading nuget... @rem NB: Must use single quotes around NUGET here, NOT double! diff --git a/Tools/msi/get_externals.bat b/Tools/msi/get_externals.bat index d66781d7c42ec49..e76ba701171c989 100644 --- a/Tools/msi/get_externals.bat +++ b/Tools/msi/get_externals.bat @@ -6,7 +6,7 @@ set HERE=%~dp0 if "%PCBUILD%"=="" (set PCBUILD=%HERE%..\..\PCbuild\) if "%EXTERNALS_DIR%"=="" (set EXTERNALS_DIR=%HERE%..\..\externals\windows-installer) if "%NUGET%"=="" (set NUGET=%EXTERNALS_DIR%\..\nuget.exe) -if "%NUGET_URL%"=="" (set NUGET_URL=https://aka.ms/nugetclidl) +if "%NUGET_URL%"=="" (set NUGET_URL=https://dist.nuget.org/win-x86-commandline/latest/nuget.exe) set DO_FETCH=true set DO_CLEAN=false From 6f9374777f392d10e1e6460887a585b5cd1dcd87 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Mon, 10 Aug 2026 09:32:59 +0200 Subject: [PATCH 066/129] [3.10] gh-146581: Fix vulnerability in shutil.unpack_archive() for ZIP files on Windows (GH-146591) (GH-149071) (#155440) [3.11] gh-146581: Fix vulnerability in shutil.unpack_archive() for ZIP files on Windows (GH-146591) (GH-149071) gh-146581: Fix vulnerability in shutil.unpack_archive() for ZIP files on Windows (GH-146591) Use ZipFile.extractall() to sanitize file names and extract files. Files with invalid names (e.g. absolute paths) are now skipped. Files containing ".." in the name are no longer skipped. (cherry picked from commit 7ef7dd0a74f47facd1fadcbc77f8fb03beb5eb4d) (cherry picked from commit fc829e88753858c8ac669594bf0093f44948c0f4) Co-authored-by: Serhiy Storchaka (cherry picked from commit 1071290169f85c09cf767c1099564a9426cdc7e8) (cherry picked from commit 906bb52ba960802614883be747cb77154f29ceea) 3.7.17.x: 3.7's _unpack_zipfile() loop is replaced the same way; test uses test.support.change_cwd. --- Lib/shutil.py | 28 +------- Lib/test/test_shutil.py | 65 +++++++++++++++++++ Lib/zipfile.py | 19 ++++-- ...-03-29-12-51-33.gh-issue-146581.4vZfB0.rst | 5 ++ 4 files changed, 87 insertions(+), 30 deletions(-) create mode 100644 Misc/NEWS.d/next/Security/2026-03-29-12-51-33.gh-issue-146581.4vZfB0.rst diff --git a/Lib/shutil.py b/Lib/shutil.py index a2e35cc80aa8a7d..0a81706544cef51 100644 --- a/Lib/shutil.py +++ b/Lib/shutil.py @@ -898,31 +898,9 @@ def _unpack_zipfile(filename, extract_dir): if not zipfile.is_zipfile(filename): raise ReadError("%s is not a zip file" % filename) - zip = zipfile.ZipFile(filename) - try: - for info in zip.infolist(): - name = info.filename - - # don't extract absolute paths or ones with .. in them - if name.startswith('/') or '..' in name: - continue - - target = os.path.join(extract_dir, *name.split('/')) - if not target: - continue - - _ensure_directory(target) - if not name.endswith('/'): - # file - data = zip.read(info.filename) - f = open(target, 'wb') - try: - f.write(data) - finally: - f.close() - del data - finally: - zip.close() + with zipfile.ZipFile(filename) as zip: + zip._ignore_invalid_names = True + zip.extractall(extract_dir) def _unpack_tarfile(filename, extract_dir, *, filter=None): """Unpack tar/tar.gz/tar.bz2/tar.xz `filename` to `extract_dir` diff --git a/Lib/test/test_shutil.py b/Lib/test/test_shutil.py index 433c55fd9a8b775..06ac223ef92a2d0 100644 --- a/Lib/test/test_shutil.py +++ b/Lib/test/test_shutil.py @@ -1299,6 +1299,71 @@ def test_unpack_archive_zip(self): with self.assertRaises(TypeError): self.check_unpack_archive('zip', filter='data') + def test_unpack_archive_zip_badpaths(self): + srcdir = self.mkdtemp() + zipname = os.path.join(srcdir, 'test.zip') + abspath = os.path.join(srcdir, 'abspath') + with zipfile.ZipFile(zipname, 'w') as zf: + zf.writestr(abspath, 'badfile') + zf.writestr(os.sep + abspath, 'badfile') + zf.writestr('/abspath', 'badfile') + zf.writestr('C:/abspath', 'badfile') + zf.writestr('D:\\abspath', 'badfile') + zf.writestr('E:abspath', 'badfile') + zf.writestr('F:/G:/abspath', 'badfile') + zf.writestr('//server/share/abspath', 'badfile') + zf.writestr('\\\\server2\\share\\abspath', 'badfile') + zf.writestr('../relpath', 'badfile') + zf.writestr(os.pardir + os.sep + 'relpath2', 'badfile') + zf.writestr('good/file', 'goodfile') + zf.writestr('good..file', 'goodfile') + + dstdir = os.path.join(self.mkdtemp(), 'dst') + unpack_archive(zipname, dstdir) + self.assertTrue(os.path.isfile(os.path.join(dstdir, 'good', 'file'))) + self.assertTrue(os.path.isfile(os.path.join(dstdir, 'good..file'))) + self.assertFalse(os.path.exists(abspath)) + self.assertFalse(os.path.exists(os.path.join(dstdir, 'abspath'))) + self.assertFalse(os.path.exists(os.path.join(dstdir, 'G_'))) + self.assertFalse(os.path.exists(os.path.join(dstdir, 'server'))) + if os.name != 'nt': + self.assertTrue(os.path.isfile(os.path.join(dstdir, 'C:', 'abspath'))) + self.assertTrue(os.path.isfile(os.path.join(dstdir, 'D:\\abspath'))) + self.assertTrue(os.path.isfile(os.path.join(dstdir, 'E:abspath'))) + self.assertTrue(os.path.isfile(os.path.join(dstdir, 'F:', 'G:', 'abspath'))) + self.assertTrue(os.path.isfile(os.path.join(dstdir, '\\\\server2\\share\\abspath'))) + if os.pardir == '..': + self.assertFalse(os.path.exists(os.path.join(dstdir, '..', 'relpath'))) + self.assertFalse(os.path.exists(os.path.join(dstdir, 'relpath'))) + else: + self.assertTrue(os.path.isfile(os.path.join(dstdir, '..', 'relpath'))) + self.assertFalse(os.path.exists(os.path.join(dstdir, os.pardir, 'relpath2'))) + self.assertFalse(os.path.exists(os.path.join(dstdir, 'relpath2'))) + + dstdir2 = os.path.join(self.mkdtemp(), 'dst') + os.mkdir(dstdir2) + with support.change_cwd(dstdir2): + unpack_archive(zipname, '') + self.assertTrue(os.path.isfile(os.path.join('good', 'file'))) + self.assertTrue(os.path.isfile('good..file')) + self.assertFalse(os.path.exists(abspath)) + self.assertFalse(os.path.exists('abspath')) + self.assertFalse(os.path.exists('C_')) + self.assertFalse(os.path.exists('server')) + if os.name != 'nt': + self.assertTrue(os.path.isfile(os.path.join('C:', 'abspath'))) + self.assertTrue(os.path.isfile('D:\\abspath')) + self.assertTrue(os.path.isfile('E:abspath')) + self.assertTrue(os.path.isfile(os.path.join('F:', 'G:', 'abspath'))) + self.assertTrue(os.path.isfile('\\\\server2\\share\\abspath')) + if os.pardir == '..': + self.assertFalse(os.path.exists(os.path.join('..', 'relpath'))) + self.assertFalse(os.path.exists('relpath')) + else: + self.assertTrue(os.path.isfile(os.path.join('..', 'relpath'))) + self.assertFalse(os.path.exists(os.path.join(os.pardir, 'relpath2'))) + self.assertFalse(os.path.exists('relpath2')) + def test_unpack_registry(self): formats = get_unpack_formats() diff --git a/Lib/zipfile.py b/Lib/zipfile.py index 5dca6eeb7d7cd8f..7ab2fbf397c1e84 100644 --- a/Lib/zipfile.py +++ b/Lib/zipfile.py @@ -1222,6 +1222,7 @@ class ZipFile: fp = None # Set here since __del__ checks it _windows_illegal_name_trans_table = None + _ignore_invalid_names = False def __init__(self, file, mode="r", compression=ZIP_STORED, allowZip64=True, compresslevel=None): @@ -1684,19 +1685,27 @@ def _extract_member(self, member, targetpath, pwd): # build the destination pathname, replacing # forward slashes to platform specific separators. - arcname = member.filename.replace('/', os.path.sep) - - if os.path.altsep: + arcname = member.filename + if os.path.sep != '/': + arcname = arcname.replace('/', os.path.sep) + if os.path.altsep and os.path.altsep != '/': arcname = arcname.replace(os.path.altsep, os.path.sep) # interpret absolute pathname as relative, remove drive letter or # UNC path, redundant separators, "." and ".." components. - arcname = os.path.splitdrive(arcname)[1] + drive, arcname = os.path.splitdrive(arcname) + if self._ignore_invalid_names and (drive or arcname.startswith(os.path.sep)): + return None + if self._ignore_invalid_names and os.path.pardir in arcname.split(os.path.sep): + return None invalid_path_parts = ('', os.path.curdir, os.path.pardir) arcname = os.path.sep.join(x for x in arcname.split(os.path.sep) if x not in invalid_path_parts) if os.path.sep == '\\': # filter illegal characters on Windows - arcname = self._sanitize_windows_name(arcname, os.path.sep) + arcname2 = self._sanitize_windows_name(arcname, os.path.sep) + if self._ignore_invalid_names and arcname2 != arcname: + return None + arcname = arcname2 targetpath = os.path.join(targetpath, arcname) targetpath = os.path.normpath(targetpath) diff --git a/Misc/NEWS.d/next/Security/2026-03-29-12-51-33.gh-issue-146581.4vZfB0.rst b/Misc/NEWS.d/next/Security/2026-03-29-12-51-33.gh-issue-146581.4vZfB0.rst new file mode 100644 index 000000000000000..98e65549d79016f --- /dev/null +++ b/Misc/NEWS.d/next/Security/2026-03-29-12-51-33.gh-issue-146581.4vZfB0.rst @@ -0,0 +1,5 @@ +Fix vulnerability in :func:`shutil.unpack_archive` for ZIP files on Windows +which allowed to write files outside of the destination tree if the patch in +the archive contains a Windows drive prefix. Now such invalid paths will be +skipped. Files containing ".." in the name (like "foo..bar") are no longer +skipped. From 369975edfdf6844344f882747fa4820f4cd029b3 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Mon, 10 Aug 2026 19:09:15 +0200 Subject: [PATCH 067/129] [3.10] gh-100372: Use BIO_eof to detect EOF for SSL_FILETYPE_ASN1 (GH-100373) (#153312) gh-100372: Use BIO_eof to detect EOF for SSL_FILETYPE_ASN1 (GH-100373) In PEM, we need to parse until error and then suppress `PEM_R_NO_START_LINE`, because PEM allows arbitrary leading and trailing data. DER, however, does not. Parsing until error and suppressing `ASN1_R_HEADER_TOO_LONG` doesn't quite work because that error also covers some cases that should be rejected. Instead, check `BIO_eof` early and stop the loop that way. (cherry picked from commit acfe02f3b05436658d92add6b168538b30f357f0) Automerge-Triggered-By: GH:Yhg1s Co-authored-by: David Benjamin Co-authored-by: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> (cherry picked from commit 129667dd0b79b271b6b8e8b00577edf97a337207) (cherry picked from commit 3eb4181a4016f3af8386e2a3ad66f495eb5593ff) 3.7.17.x: ported onto 3.7's _add_ca_certs() (retval starts at 0; no loaded == 0 branch). --- Lib/test/test_ssl.py | 2 ++ .../2022-12-20-10-55-14.gh-issue-100372.utfP65.rst | 2 ++ Modules/_ssl.c | 9 ++++++--- 3 files changed, 10 insertions(+), 3 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2022-12-20-10-55-14.gh-issue-100372.utfP65.rst diff --git a/Lib/test/test_ssl.py b/Lib/test/test_ssl.py index 952ddae0946e937..cdd3f00431ca245 100644 --- a/Lib/test/test_ssl.py +++ b/Lib/test/test_ssl.py @@ -1495,6 +1495,8 @@ def test_load_verify_cadata(self): ctx.load_verify_locations(cadata="broken") with self.assertRaisesRegex(ssl.SSLError, "not enough data"): ctx.load_verify_locations(cadata=b"broken") + with self.assertRaises(ssl.SSLError): + ctx.load_verify_locations(cadata=cacert_der + b"A") def test_load_dh_params(self): diff --git a/Misc/NEWS.d/next/Library/2022-12-20-10-55-14.gh-issue-100372.utfP65.rst b/Misc/NEWS.d/next/Library/2022-12-20-10-55-14.gh-issue-100372.utfP65.rst new file mode 100644 index 000000000000000..ec37aff5092c3af --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-12-20-10-55-14.gh-issue-100372.utfP65.rst @@ -0,0 +1,2 @@ +:meth:`ssl.SSLContext.load_verify_locations` no longer incorrectly accepts +some cases of trailing data when parsing DER. diff --git a/Modules/_ssl.c b/Modules/_ssl.c index cdff4ebbbe9e26a..0254030d4f51fe2 100644 --- a/Modules/_ssl.c +++ b/Modules/_ssl.c @@ -3976,7 +3976,7 @@ _add_ca_certs(PySSLContext *self, void *data, Py_ssize_t len, { BIO *biobuf = NULL; X509_STORE *store; - int retval = 0, err, loaded = 0; + int retval = 0, err, loaded = 0, was_bio_eof = 0; assert(filetype == SSL_FILETYPE_ASN1 || filetype == SSL_FILETYPE_PEM); @@ -4004,6 +4004,10 @@ _add_ca_certs(PySSLContext *self, void *data, Py_ssize_t len, int r; if (filetype == SSL_FILETYPE_ASN1) { + if (BIO_eof(biobuf)) { + was_bio_eof = 1; + break; + } cert = d2i_X509_bio(biobuf, NULL); } else { cert = PEM_read_bio_X509(biobuf, NULL, @@ -4032,8 +4036,7 @@ _add_ca_certs(PySSLContext *self, void *data, Py_ssize_t len, err = ERR_peek_last_error(); if ((filetype == SSL_FILETYPE_ASN1) && (loaded > 0) && - (ERR_GET_LIB(err) == ERR_LIB_ASN1) && - (ERR_GET_REASON(err) == ASN1_R_HEADER_TOO_LONG)) { + was_bio_eof) { /* EOF ASN1 file, not an error */ ERR_clear_error(); retval = 0; From 32e7e3dee812317f820d4f9b1fd92aed6aea7db6 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Mon, 10 Aug 2026 21:53:37 +0200 Subject: [PATCH 068/129] [3.10] gh-149079: Fix O(n^2) canonical ordering in unicodedata.normalize() (GH-149080) (GH-150843) (#155159) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit [3.12] gh-149079: Fix O(n^2) canonical ordering in unicodedata.normalize() (GH-149080) (GH-150843) Replace the insertion sort used for canonical ordering of combining characters with a hybrid approach: insertion sort for short runs (< 20) and counting sort for longer runs, reducing worst-case complexity from O(n^2) to O(n). This prevents denial of service via crafted Unicode strings with many combining characters in alternating CCC order. (cherry picked from commit 991224b1e8311c85f198f6dd8208bf8cff7fc26f) (cherry picked from commit d3ab945af25b28dfe13ac6cb40c124a01b33ce1f) Co-authored-by: Petr Viktorin Co-authored-by: Seth Larson Co-authored-by: ch4n3-yoon Co-authored-by: Seokchan Yoon <13852925+ch4n3-yoon@users.noreply.github.com> Co-authored-by: Stan Ulbrych Co-authored-by: Bénédikt Tran <10796600+picnixz@users.noreply.github.com> Co-authored-by: Serhiy Storchaka Co-authored-by: Maurycy Pawłowski-Wieroński Co-authored-by: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> (cherry picked from commit db744c0776c1d5dd11aaa70eff2a6993c408bacc) (cherry picked from commit f34098bf33127f3714c5d4076f51d9cf306567f1) --- Lib/test/test_unicodedata.py | 28 ++++ ...-04-27-16-36-11.gh-issue-149079.vKl-LM.rst | 5 + Modules/unicodedata.c | 144 ++++++++++++++---- 3 files changed, 151 insertions(+), 26 deletions(-) create mode 100644 Misc/NEWS.d/next/Security/2026-04-27-16-36-11.gh-issue-149079.vKl-LM.rst diff --git a/Lib/test/test_unicodedata.py b/Lib/test/test_unicodedata.py index 170778fa977db1b..b5af91b4acc602d 100644 --- a/Lib/test/test_unicodedata.py +++ b/Lib/test/test_unicodedata.py @@ -208,6 +208,34 @@ def test_issue10254(self): b = 'C\u0338' * 20 + '\xC7' self.assertEqual(self.db.normalize('NFC', a), b) + def test_long_combining_mark_run(self): + # gh-149079: avoid quadratic canonical ordering. + payload = "a" + ("\u0300\u0327" * 32) + nfd = "a" + ("\u0327" * 32) + ("\u0300" * 32) + nfc = "\u00e0" + ("\u0327" * 32) + ("\u0300" * 31) + + self.assertEqual(self.db.normalize("NFD", payload), nfd) + self.assertEqual(self.db.normalize("NFKD", payload), nfd) + self.assertEqual(self.db.normalize("NFC", payload), nfc) + self.assertEqual(self.db.normalize("NFKC", payload), nfc) + + def test_combining_mark_run_fast_paths(self): + # gh-149079: cover short runs and already-sorted long runs. + short_payload = "a" + ("\u0300\u0327" * 9) + "\u0300" + short_nfd = "a" + ("\u0327" * 9) + ("\u0300" * 10) + short_nfc = "\u00e0" + ("\u0327" * 9) + ("\u0300" * 9) + long_sorted = "a" + ("\u0327" * 30) + ("\u0300" * 30) + long_sorted_nfc = "\u00e0" + ("\u0327" * 30) + ("\u0300" * 29) + + self.assertEqual(self.db.normalize("NFD", short_payload), short_nfd) + self.assertEqual(self.db.normalize("NFKD", short_payload), short_nfd) + self.assertEqual(self.db.normalize("NFC", short_payload), short_nfc) + self.assertEqual(self.db.normalize("NFKC", short_payload), short_nfc) + self.assertEqual(self.db.normalize("NFD", long_sorted), long_sorted) + self.assertEqual(self.db.normalize("NFKD", long_sorted), long_sorted) + self.assertEqual(self.db.normalize("NFC", long_sorted), long_sorted_nfc) + self.assertEqual(self.db.normalize("NFKC", long_sorted), long_sorted_nfc) + def test_issue29456(self): # Fix #29456 u1176_str_a = '\u1100\u1176\u11a8' diff --git a/Misc/NEWS.d/next/Security/2026-04-27-16-36-11.gh-issue-149079.vKl-LM.rst b/Misc/NEWS.d/next/Security/2026-04-27-16-36-11.gh-issue-149079.vKl-LM.rst new file mode 100644 index 000000000000000..4ed22b58f7405f5 --- /dev/null +++ b/Misc/NEWS.d/next/Security/2026-04-27-16-36-11.gh-issue-149079.vKl-LM.rst @@ -0,0 +1,5 @@ +Fix a potential denial of service in :func:`unicodedata.normalize`. The +canonical ordering step of Unicode normalization used a quadratic-time insertion +sort for reordering combining characters, which could be exploited with +crafted input containing many combining characters in non-canonical order. +Replaced with a linear-time counting sort for long runs. diff --git a/Modules/unicodedata.c b/Modules/unicodedata.c index e8788f5036ddefe..b69f8a9c87283ed 100644 --- a/Modules/unicodedata.c +++ b/Modules/unicodedata.c @@ -482,19 +482,80 @@ get_decomp_record(PyObject *self, Py_UCS4 code, int *index, int *prefix, int *co #define NCount (VCount*TCount) #define SCount (LCount*NCount) +/* Small combining runs are usually cheaper with insertion sort. */ +#define CANONICAL_ORDERING_COUNTING_SORT_THRESHOLD 20 + +static void +canonical_ordering_sort_insertion(int kind, void *data, + Py_ssize_t start, Py_ssize_t end) +{ + for (Py_ssize_t i = start + 1; i < end; i++) { + Py_UCS4 code = PyUnicode_READ(kind, data, i); + unsigned char combining = _getrecord_ex(code)->combining; + Py_ssize_t j = i; + + while (j > start) { + Py_UCS4 previous = PyUnicode_READ(kind, data, j - 1); + if (_getrecord_ex(previous)->combining <= combining) { + break; + } + PyUnicode_WRITE(kind, data, j, previous); + j--; + } + if (j != i) { + PyUnicode_WRITE(kind, data, j, code); + } + } +} + +static void +canonical_ordering_sort_counting(int kind, void *data, + Py_ssize_t start, Py_ssize_t end, + Py_UCS4 *sortbuf) +{ + Py_ssize_t counts[256] = {0}; + Py_ssize_t run_length = end - start; + Py_ssize_t total = 0; + + for (Py_ssize_t i = start; i < end; i++) { + Py_UCS4 code = PyUnicode_READ(kind, data, i); + unsigned char combining = _getrecord_ex(code)->combining; + counts[combining]++; + } + + for (size_t i = 0; i < Py_ARRAY_LENGTH(counts); i++) { + Py_ssize_t count = counts[i]; + counts[i] = total; + total += count; + } + + /* Reuse counts[] as the next output slot for each CCC. */ + for (Py_ssize_t i = start; i < end; i++) { + Py_UCS4 code = PyUnicode_READ(kind, data, i); + unsigned char combining = _getrecord_ex(code)->combining; + sortbuf[counts[combining]++] = code; + } + for (Py_ssize_t i = 0; i < run_length; i++) { + PyUnicode_WRITE(kind, data, start + i, sortbuf[i]); + } +} + static PyObject* nfd_nfkd(PyObject *self, PyObject *input, int k) { PyObject *result; Py_UCS4 *output; Py_ssize_t i, o, osize; - int kind; - void *data; + int input_kind, result_kind; + const void *input_data; + void *result_data; /* Longest decomposition in Unicode 3.2: U+FDFA */ Py_UCS4 stack[20]; Py_ssize_t space, isize; int index, prefix, count, stackptr; unsigned char prev, cur; + Py_UCS4 *sortbuf = NULL; + Py_ssize_t sortbuflen = 0; stackptr = 0; isize = PyUnicode_GET_LENGTH(input); @@ -514,11 +575,11 @@ nfd_nfkd(PyObject *self, PyObject *input, int k) return NULL; } i = o = 0; - kind = PyUnicode_KIND(input); - data = PyUnicode_DATA(input); + input_kind = PyUnicode_KIND(input); + input_data = PyUnicode_DATA(input); while (i < isize) { - stack[stackptr++] = PyUnicode_READ(kind, data, i++); + stack[stackptr++] = PyUnicode_READ(input_kind, input_data, i++); while(stackptr) { Py_UCS4 code = stack[--stackptr]; /* Hangul Decomposition adds three characters in @@ -583,35 +644,66 @@ nfd_nfkd(PyObject *self, PyObject *input, int k) PyMem_Free(output); if (!result) return NULL; + /* result is guaranteed to be ready, as it is compact. */ - kind = PyUnicode_KIND(result); - data = PyUnicode_DATA(result); + result_kind = PyUnicode_KIND(result); + result_data = PyUnicode_DATA(result); - /* Sort canonically. */ + /* Sort each consecutive combining-character run canonically. */ i = 0; - prev = _getrecord_ex(PyUnicode_READ(kind, data, i))->combining; - for (i++; i < PyUnicode_GET_LENGTH(result); i++) { - cur = _getrecord_ex(PyUnicode_READ(kind, data, i))->combining; - if (prev == 0 || cur == 0 || prev <= cur) { - prev = cur; + while (i < o) { + Py_ssize_t run_length, run_start; + int needs_sort = 0; + + Py_UCS4 ch = PyUnicode_READ(result_kind, result_data, i); + prev = _getrecord_ex(ch)->combining; + if (prev == 0) { + i++; continue; } - /* Non-canonical order. Need to switch *i with previous. */ - o = i - 1; - while (1) { - Py_UCS4 tmp = PyUnicode_READ(kind, data, o+1); - PyUnicode_WRITE(kind, data, o+1, - PyUnicode_READ(kind, data, o)); - PyUnicode_WRITE(kind, data, o, tmp); - o--; - if (o < 0) - break; - prev = _getrecord_ex(PyUnicode_READ(kind, data, o))->combining; - if (prev == 0 || prev <= cur) + + run_start = i++; + while (i < o) { + Py_UCS4 ch = PyUnicode_READ(result_kind, result_data, i); + cur = _getrecord_ex(ch)->combining; + if (cur == 0) { break; + } + if (prev > cur) { + needs_sort = 1; + } + prev = cur; + i++; } - prev = _getrecord_ex(PyUnicode_READ(kind, data, i))->combining; + if (!needs_sort) { + continue; + } + + run_length = i - run_start; + if (run_length < CANONICAL_ORDERING_COUNTING_SORT_THRESHOLD) { + canonical_ordering_sort_insertion(result_kind, result_data, + run_start, i); + continue; + } + + if (run_length > sortbuflen) { + Py_UCS4 *new_sortbuf = PyMem_Resize(sortbuf, + Py_UCS4, + run_length); + if (new_sortbuf == NULL) { + PyErr_NoMemory(); + PyMem_Free(sortbuf); + Py_DECREF(result); + return NULL; + } + sortbuf = new_sortbuf; + sortbuflen = run_length; + } + + canonical_ordering_sort_counting(result_kind, result_data, + run_start, i, sortbuf); } + PyMem_Free(sortbuf); return result; } From 7d7ab9462a8de67eaa15cb164403714d2cb52888 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Mon, 10 Aug 2026 21:53:42 +0200 Subject: [PATCH 069/129] [3.10] gh-109638: Fix exponential time in csv.Sniffer for doubled quotes (GH-154868) (GH-155113) (#155116) * [3.13] gh-109638: Fix exponential time in csv.Sniffer for doubled quotes (GH-154868) (GH-155113) (cherry picked from commit eaa9bac70343f2e0a8499ac4a1894ef7608ef0fa) Co-authored-by: Serhiy Storchaka (cherry picked from commit d52184ba1ed0e7c0fe8afe4928fbb72a0f9526fb) * Match the body of a quoted field without possessive quantifiers They were added in 3.11. An unrolled loop is unambiguous at every position, so it does not backtrack either. Co-Authored-By: Claude Opus 5 (1M context) --------- Co-authored-by: Serhiy Storchaka Co-authored-by: Claude Opus 5 (1M context) Co-authored-by: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> (cherry picked from commit 19e5024518f8a9d6e3b5a90e09d4536fc2864bed) (cherry picked from commit 50250392897af326df3e1d708f22624297058765) --- Lib/csv.py | 32 +++++++++----- Lib/test/test_csv.py | 44 +++++++++++++++++++ ...-07-29-11-25-00.gh-issue-109638.Vt2Rn9.rst | 3 ++ 3 files changed, 67 insertions(+), 12 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2026-07-29-11-25-00.gh-issue-109638.Vt2Rn9.rst diff --git a/Lib/csv.py b/Lib/csv.py index 58624af9053493b..b7fd938bd76b120 100644 --- a/Lib/csv.py +++ b/Lib/csv.py @@ -178,6 +178,8 @@ def sniff(self, sample, delimiters=None): Returns a dialect (or None) corresponding to the sample """ + sample = sample.replace('\r\n', '\n').replace('\r', '\n') + quotechar, doublequote, delimiter, skipinitialspace = \ self._guess_quote_and_delimiter(sample, delimiters) if not delimiter: @@ -262,18 +264,24 @@ def _guess_quote_and_delimiter(self, data, delimiters): delim = '' skipinitialspace = 0 - # if we see an extra quote between delimiters, we've got a - # double quoted format - dq_regexp = re.compile( - r"((%(delim)s)|^)\W*%(quote)s[^%(delim)s\n]*%(quote)s[^%(delim)s\n]*%(quote)s\W*((%(delim)s)|$)" % \ - {'delim':re.escape(delim), 'quote':quotechar}, re.MULTILINE) - - - - if dq_regexp.search(data): - doublequote = True - else: - doublequote = False + # A doubled quote character inside a quoted field means + # a double quoted format. Match whole fields, so that a match + # cannot slide across field boundaries. + doublequote = False + if delim: + dq_regexp = re.compile( + r"(?:(?<=%(delim)s)|^)%(space)s%(quote)s" # ," + # The body, as an unrolled loop: it is unambiguous, + # so it does not backtrack. + r"([^%(quote)s]*(?:%(quote)s%(quote)s[^%(quote)s]*)*)" + r"%(quote)s(?:%(delim)s|$)" # ", + % {'delim': re.escape(delim), 'quote': quotechar, + # Skipping spaces after a space rescans them. + 'space': ' *' if delim != ' ' else ''}, + re.MULTILINE) + dquotechar = quotechar * 2 + doublequote = any(dquotechar in m[1] + for m in dq_regexp.finditer(data)) return (quotechar, doublequote, delim, skipinitialspace) diff --git a/Lib/test/test_csv.py b/Lib/test/test_csv.py index b65cbf6a50e3ec3..9149ab04a0533c7 100644 --- a/Lib/test/test_csv.py +++ b/Lib/test/test_csv.py @@ -1047,6 +1047,50 @@ def test_doublequote(self): dialect = sniffer.sniff(self.sample9) self.assertTrue(dialect.doublequote) + def test_sniff_regex_backtracking(self): + # gh-109638: this artificial sample used to take minutes. + sniffer = csv.Sniffer() + sample = '"",' * 100 + '"' * 100 + '0' + '"' * 100 + '0' + self.assertEqual(sniffer.sniff(sample).delimiter, ',') + + def test_sniff_doublequote_across_fields(self): + # A quoted field which contains the delimiter, followed by + # an empty quoted field, is not a doubled quote. + sniffer = csv.Sniffer() + sample = '",","",","\n' * 4 + dialect = sniffer.sniff(sample) + self.assertEqual(dialect.delimiter, ',') + self.assertEqual(dialect.quotechar, '"') + self.assertIs(dialect.doublequote, False) + self.assertEqual(next(csv.reader(StringIO(sample), dialect)), + [',', '', ',']) + + def test_sniff_doublequote_record_separators(self): + # The record separator ends a field as a delimiter does. + sniffer = csv.Sniffer() + for sep in '\n', '\r\n', '\r': + with self.subTest(sep=sep): + sample = ('x,"a""b"' + sep + 'y,"c"' + sep) * 2 + self.assertIs(sniffer.sniff(sample).doublequote, True) + sample = ('"",","' + sep) * 4 + self.assertIs(sniffer.sniff(sample).doublequote, False) + + def test_sniff_single_column(self): + # This sample used to be quadratic. + sniffer = csv.Sniffer() + sample = '"a"\n' + ' ' * 100000 + with self.assertRaisesRegex(csv.Error, "Could not determine delimiter"): + sniffer.sniff(sample, delimiters=',;') + + def test_sniff_space_delimiter(self): + # This sample used to be quadratic. + sniffer = csv.Sniffer() + sample = '"a" "b"\n' + ' ' * 100000 + dialect = sniffer.sniff(sample) + self.assertEqual(dialect.delimiter, ' ') + self.assertIs(dialect.doublequote, False) + + class NUL: def write(s, *args): pass diff --git a/Misc/NEWS.d/next/Library/2026-07-29-11-25-00.gh-issue-109638.Vt2Rn9.rst b/Misc/NEWS.d/next/Library/2026-07-29-11-25-00.gh-issue-109638.Vt2Rn9.rst new file mode 100644 index 000000000000000..f44163a20c8278e --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-07-29-11-25-00.gh-issue-109638.Vt2Rn9.rst @@ -0,0 +1,3 @@ +Fix exponential time in :meth:`csv.Sniffer.sniff` for a sample which contains +many quote characters. A doubled quote character is now also detected in +a field which contains the delimiter or a line break. From b1067396f17fb2561e2594b3923be4f80bee258c Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Mon, 10 Aug 2026 21:53:46 +0200 Subject: [PATCH 070/129] [3.10] gh-152674: Avoid quadratic behavior in xml.etree.ElementPath index predicates (GH-152676) (#154814) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit gh-152674: Avoid quadratic behavior in xml.etree.ElementPath index predicates (GH-152676) (cherry picked from commit 2ffab083782968a4d732738f4f1dff6bbd69d2b0) Co-authored-by: Petr Viktorin Co-authored-by: Alexis Co-authored-by: Bénédikt Tran <10796600+picnixz@users.noreply.github.com> Co-authored-by: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> (cherry picked from commit 037965c00a427cba5c05447efadc67c51a492e85) (cherry picked from commit cc4db8cbf6140928d613c81cd51a321d6a5cccf9) --- Lib/test/test_xml_etree.py | 31 +++++++++++++++++++ Lib/xml/etree/ElementPath.py | 17 +++++++--- ...-06-30-13-24-13.gh-issue-152674.-2QVoL.rst | 6 ++++ 3 files changed, 49 insertions(+), 5 deletions(-) create mode 100644 Misc/NEWS.d/next/Security/2026-06-30-13-24-13.gh-issue-152674.-2QVoL.rst diff --git a/Lib/test/test_xml_etree.py b/Lib/test/test_xml_etree.py index 0cf38b13c24813b..fd229d4bdbd5925 100644 --- a/Lib/test/test_xml_etree.py +++ b/Lib/test/test_xml_etree.py @@ -2277,6 +2277,37 @@ def test_find_xpath(self): self.assertRaisesRegex(SyntaxError, 'XPath', e.find, './tag[last()-0]') self.assertRaisesRegex(SyntaxError, 'XPath', e.find, './tag[last()+1]') + def test_find_xpath_index_no_quadratic_complexity(self): + class CountingElement(ET.Element): + findall_calls = 0 + def findall(self, *args, **kwargs): + type(self).findall_calls += 1 + return super().findall(*args, **kwargs) + + def work(n, pattern): + root = CountingElement("root") + for _ in range(n): + ET.SubElement(root, "a") + CountingElement.findall_calls = 0 + root.findall(pattern) + return CountingElement.findall_calls + + for pattern in [".//a[1]", ".//a[last()]"]: + w1 = work(1024, pattern) + w2 = work(2048, pattern) + w3 = work(4096, pattern) + + self.assertGreater(w1, 0) + r1 = w2 / w1 + r2 = w3 / w2 + # Doubling N must not ~double the parent.findall calls. + # Linear-in-N call counts indicate the cache is missing. + self.assertLess( + max(r1, r2), 1.5, + msg=f"Possible quadratic behavior on {pattern!r}: " + f"calls={w1, w2, w3} ratios={r1, r2}", + ) + def test_findall(self): e = ET.XML(SAMPLE_XML) e[2] = ET.XML(SAMPLE_SECTION) diff --git a/Lib/xml/etree/ElementPath.py b/Lib/xml/etree/ElementPath.py index ef32917b14d41e2..24d78bfdd4a2965 100644 --- a/Lib/xml/etree/ElementPath.py +++ b/Lib/xml/etree/ElementPath.py @@ -229,15 +229,22 @@ def select(context, result): index = -1 def select(context, result): parent_map = get_parent_map(context) + cache = {} for elem in result: try: parent = parent_map[elem] + except KeyError: + continue + key = (parent, elem.tag) + if key not in cache: # FIXME: what if the selector is "*" ? - elems = list(parent.findall(elem.tag)) - if elems[index] is elem: - yield elem - except (IndexError, KeyError): - pass + elems = parent.findall(elem.tag) + try: + cache[key] = elems[index] + except IndexError: + cache[key] = None + if cache[key] is elem: + yield elem return select raise SyntaxError("invalid predicate") diff --git a/Misc/NEWS.d/next/Security/2026-06-30-13-24-13.gh-issue-152674.-2QVoL.rst b/Misc/NEWS.d/next/Security/2026-06-30-13-24-13.gh-issue-152674.-2QVoL.rst new file mode 100644 index 000000000000000..69e73008af79f14 --- /dev/null +++ b/Misc/NEWS.d/next/Security/2026-06-30-13-24-13.gh-issue-152674.-2QVoL.rst @@ -0,0 +1,6 @@ +The :class:`xml.etree.ElementTree.Element` methods +:meth:`~xml.etree.ElementTree.Element.findall`, +:meth:`~xml.etree.ElementTree.Element.iterfind` and +:meth:`~xml.etree.ElementTree.Element.find` avoid quadratic behavior when +using XPath index predicates (``[1]``, ``[last()]``, ``[last()-N]``) on XML +documents with many same-tag siblings. From 97d38906c0c501c919bbed8b5d12943431a48e0f Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Mon, 10 Aug 2026 22:53:50 +0300 Subject: [PATCH 071/129] [3.10] gh-153030: Fix quadratic complexity in incremental parsing in HTMLParser (GH-153031) (#153043) gh-153030: Fix quadratic complexity in incremental parsing in HTMLParser (GH-153031) When an unterminated construct (e.g. a tag or comment) spanned many feed() calls, rescanning the growing buffer and concatenating new data onto it were both quadratic. New data is now accumulated in a list and only joined and parsed once enough has piled up. (cherry picked from commit bcf98ddbc40ec9b3ee87da0124a5660b19b7e606) Co-authored-by: Claude Opus 4.8 (cherry picked from commit 1e7956f1a722df9aabc509c30f8fbdc3a2b4fdc7) (cherry picked from commit 27e2b0a7da5d6745da4f289292ba2bcb3eddf195) --- Lib/html/parser.py | 32 +++++++++++++++++-- Lib/test/test_htmlparser.py | 20 ++++++++++++ ...-07-04-17-00-00.gh-issue-153030.RovkP6.rst | 3 ++ 3 files changed, 53 insertions(+), 2 deletions(-) create mode 100644 Misc/NEWS.d/next/Security/2026-07-04-17-00-00.gh-issue-153030.RovkP6.rst diff --git a/Lib/html/parser.py b/Lib/html/parser.py index 991b17bfdd02fde..b0b474bfa3683d5 100644 --- a/Lib/html/parser.py +++ b/Lib/html/parser.py @@ -130,6 +130,9 @@ def reset(self): self.cdata_elem = None self._support_cdata = True self._escapable = True + self._pending = [] + self._pending_len = 0 + self._parse_threshold = 1 _markupbase.ParserBase.reset(self) def feed(self, data): @@ -138,11 +141,36 @@ def feed(self, data): Call this as often as you want, with as little or as much text as you want (may include '\n'). """ - self.rawdata = self.rawdata + data - self.goahead(0) + # Accumulate new data in a list and only join and parse it once + # enough has piled up. Rescanning an unparsed buffer (e.g. an + # unterminated tag) and concatenating onto it on every call would + # both be quadratic in the input size. + self._pending_len += len(data) + if self._pending_len < self._parse_threshold: + self._pending.append(data) + else: + if not self._pending: + self.rawdata += data + else: + self._pending.append(data) + self.rawdata += ''.join(self._pending) + self._pending.clear() + self._pending_len = 0 + n = len(self.rawdata) + self.goahead(0) + if len(self.rawdata) < n: + # Some data was parsed; resume on the next call. + self._parse_threshold = 1 + else: + # Nothing was parsed; wait until the buffer doubles. + self._parse_threshold = len(self.rawdata) def close(self): """Handle any buffered data.""" + if self._pending: + self.rawdata += ''.join(self._pending) + self._pending.clear() + self._pending_len = 0 self.goahead(1) __starttag_text = None diff --git a/Lib/test/test_htmlparser.py b/Lib/test/test_htmlparser.py index b95cdc1e3311d9b..0cd63bac3f25ade 100644 --- a/Lib/test/test_htmlparser.py +++ b/Lib/test/test_htmlparser.py @@ -945,6 +945,26 @@ def check(source): check("") # comment + check("") # processing instruction + check("") # doctype + check("") # CDATA section + check("") # start tag + check("") # RAWTEXT element + class AttributesTestCase(TestCaseBase): diff --git a/Misc/NEWS.d/next/Security/2026-07-04-17-00-00.gh-issue-153030.RovkP6.rst b/Misc/NEWS.d/next/Security/2026-07-04-17-00-00.gh-issue-153030.RovkP6.rst new file mode 100644 index 000000000000000..d1d60593f4ba7d2 --- /dev/null +++ b/Misc/NEWS.d/next/Security/2026-07-04-17-00-00.gh-issue-153030.RovkP6.rst @@ -0,0 +1,3 @@ +Fixed quadratic complexity in incremental parsing of long unterminated +constructs (such as tags or comments) in :class:`html.parser.HTMLParser`, +which could be exploited for a denial of service. From d732baf96b8ab01bf7815021304b26052a0f09b9 Mon Sep 17 00:00:00 2001 From: Stan Ulbrych Date: Mon, 10 Aug 2026 20:53:54 +0100 Subject: [PATCH 072/129] [3.10] gh-151558: Fix symlink escape via `tarfile` hardlink-extraction fallback (GH-151559) (#152080) gh-151558: Fix symlink escape via `tarfile` hardlink-extraction fallback (GH-151559) (cherry picked from commit 27dd970bf6b17ebca7c8ed486a40ab043ed7af8f) (cherry picked from commit 0f852b3f07dd8e71e40326a51c02afbf16a42cc5) (cherry picked from commit ffd10db259017b8bc2c96bb94edd07e60e0c880b) --- Lib/tarfile.py | 3 +++ Lib/test/test_tarfile.py | 23 +++++++++++++++++++ ...-06-10-13-08-19.gh-issue-151558.mL74i2.rst | 3 +++ 3 files changed, 29 insertions(+) create mode 100644 Misc/NEWS.d/next/Security/2026-06-10-13-08-19.gh-issue-151558.mL74i2.rst diff --git a/Lib/tarfile.py b/Lib/tarfile.py index 16736ebe3737f18..49ed25cf13dfc3e 100755 --- a/Lib/tarfile.py +++ b/Lib/tarfile.py @@ -2621,6 +2621,9 @@ def makelink_with_filter(self, tarinfo, targetpath, "makelink_with_filter: if filter_function is not None, " + "extraction_root must also not be None") try: + filter_function( + unfiltered.replace(name=tarinfo.name, deep=False), + extraction_root) filtered = filter_function(unfiltered, extraction_root) except _FILTER_ERRORS as cause: raise LinkFallbackError(tarinfo, unfiltered.name) from cause diff --git a/Lib/test/test_tarfile.py b/Lib/test/test_tarfile.py index 897a518158c7e2f..5ad04849a3e8cb4 100644 --- a/Lib/test/test_tarfile.py +++ b/Lib/test/test_tarfile.py @@ -3772,6 +3772,29 @@ def test_sneaky_hardlink_fallback(self): self.expect_file("boom", symlink_to='../../link_here') self.expect_file("c", symlink_to='b') + def test_sneaky_hardlink_fallback_deep(self): + # (CVE-2026-11940) + with ArchiveMaker() as arc: + arc.add("a/b/s", symlink_to=os.path.join("..", "escape")) + arc.add("s", hardlink_to=os.path.join("a", "b", "s")) + + with self.check_context(arc.open(), 'data'): + e = self.expect_exception( + tarfile.LinkFallbackError, + "link 's' would be extracted as a copy of " + + "'a/b/s', which was rejected") + self.assertIsInstance(e.__cause__, + tarfile.LinkOutsideDestinationError) + + for filter in 'tar', 'fully_trusted': + with self.subTest(filter), self.check_context(arc.open(), filter): + if not os_helper.can_symlink(): + self.expect_file("a/") + self.expect_file("a/b/") + else: + self.expect_file("a/b/s", symlink_to=os.path.join('..', 'escape')) + self.expect_file("s", symlink_to=os.path.join('..', 'escape')) + def test_exfiltration_via_symlink(self): # (CVE-2025-4138) # Test changing symlinks that result in a symlink pointing outside diff --git a/Misc/NEWS.d/next/Security/2026-06-10-13-08-19.gh-issue-151558.mL74i2.rst b/Misc/NEWS.d/next/Security/2026-06-10-13-08-19.gh-issue-151558.mL74i2.rst new file mode 100644 index 000000000000000..575081600f8bdd9 --- /dev/null +++ b/Misc/NEWS.d/next/Security/2026-06-10-13-08-19.gh-issue-151558.mL74i2.rst @@ -0,0 +1,3 @@ +Fixed an vulnerability in the :mod:`tarfile` ``data`` and ``tar`` extraction +filters where crafted archives could create a symlink pointing outside the +destination directory. This was a bypass of CVE-2025-4330. From d050ade3884cbaa2da15f31a311cf3d5f58eca52 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Mon, 10 Aug 2026 21:53:58 +0200 Subject: [PATCH 073/129] [3.10] gh-146333: Fix quadratic regex backtracking in configparser option parsing (GH-146399) (GH-148559) (#154878) Use negative lookahead in option regex to prevent backtracking, and to avoid changing logic outside the regexes (since people could use the regex directly). (cherry picked from commit 7e0a0be4097f9d29d66fe23f5af86f18a34ed7dd) (cherry picked from commit a5969e8f0fda37aaf0e2f844fdcfca9d822a70b1) Co-authored-by: Joshua Swanson <22283299+joshuaswanson@users.noreply.github.com> Co-authored-by: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> (cherry picked from commit 0e14e5db4ccbcdd339aa0ccf81ca1fa5c4c2f6b6) (cherry picked from commit 76ac101d2c2f2d3300fe3cce588e2710f351ed64) --- Lib/configparser.py | 8 ++++++-- Lib/test/test_configparser.py | 20 +++++++++++++++++++ ...3-25-00-51-03.gh-issue-146333.LqdL__bn.rst | 3 +++ 3 files changed, 29 insertions(+), 2 deletions(-) create mode 100644 Misc/NEWS.d/next/Security/2026-03-25-00-51-03.gh-issue-146333.LqdL__bn.rst diff --git a/Lib/configparser.py b/Lib/configparser.py index 4caf3faadc7585e..14f10d3469d6fb4 100644 --- a/Lib/configparser.py +++ b/Lib/configparser.py @@ -566,7 +566,9 @@ class RawConfigParser(MutableMapping): \] # ] """ _OPT_TMPL = r""" - (?P