2.7.18.16 Release: 17 security fixes, bundled Expat 2.9.0

Doc/library/os.rst

@@ -1233,11 +1233,18 @@ Files and Directories
    platform-dependent.  On some platforms, they are ignored and you should call
    :func:`chmod` explicitly to set them.
 
+   On Windows, a *mode* of ``0700`` is specifically handled to apply access
+   control to the new directory such that only the current user and
+   administrators have access. Other values of *mode* are ignored.
+
    It is also possible to create temporary directories; see the
    :mod:`tempfile` module's :func:`tempfile.mkdtemp` function.
 
    Availability: Unix, Windows.
 
+   .. versionchanged:: 2.7.18.16
+      Windows now handles a *mode* of ``0700``.
+
 
 .. function:: makedirs(path[, mode])
 

Include/patchlevel.h

@@ -27,7 +27,7 @@
 #define PY_RELEASE_SERIAL	0
 
 /* Version as a string */
-#define PY_VERSION      	"2.7.18.15"
+#define PY_VERSION      	"2.7.18.16"
 /*--end constants--*/
 
 /* Subversion Revision number of this file (not of the repository). Empty

Include/pyexpat.h

@@ -45,6 +45,9 @@ struct PyExpat_CAPI
     void (*SetUserData)(XML_Parser parser, void *userData);
     /* might be none for expat < 2.1.0 */
     int (*SetHashSalt)(XML_Parser parser, unsigned long hash_salt);
+    /* might be NULL for expat < 2.8.0 */
+    XML_Bool (*SetHashSalt16Bytes)(
+        XML_Parser parser, const uint8_t entropy[16]);
     /* always add new stuff to the end! */
 };
 

Lib/BaseHTTPServer.py

@@ -287,6 +287,13 @@ def parse_request(self):
             return False
         self.command, self.path, self.request_version = command, path, version
 
+        # gh-87389: The purpose of replacing '//' with '/' is to protect
+        # against open redirect attacks possibly triggered if the path starts
+        # with '//' because http clients treat //path as an absolute URI
+        # without scheme (similar to http://path) rather than a path.
+        if self.path.startswith('//'):
+            self.path = '/' + self.path.lstrip('/')  # Reduce to a single /
+
         # Examine the headers and look for a Connection directive
         self.headers = self.MessageClass(self.rfile, 0)
 

Lib/HTMLParser.py

@@ -106,6 +106,9 @@ def reset(self):
         self.lasttag = '???'
         self.interesting = interesting_normal
         self.cdata_elem = None
+        self._pending = []
+        self._pending_len = 0
+        self._parse_threshold = 1
         markupbase.ParserBase.reset(self)
 
     def feed(self, data):
@@ -114,11 +117,36 @@ def feed(self, data):
         Call this as often as you want, with as little or as much text
         as you want (may include '\n').
         """
-        self.rawdata = self.rawdata + data
-        self.goahead(0)
+        # Accumulate new data in a list and only join and parse it once
+        # enough has piled up.  Rescanning an unparsed buffer (e.g. an
+        # unterminated tag) and concatenating onto it on every call would
+        # both be quadratic in the input size.
+        self._pending_len += len(data)
+        if self._pending_len < self._parse_threshold:
+            self._pending.append(data)
+        else:
+            if not self._pending:
+                self.rawdata += data
+            else:
+                self._pending.append(data)
+                self.rawdata += ''.join(self._pending)
+                del self._pending[:]
+            self._pending_len = 0
+            n = len(self.rawdata)
+            self.goahead(0)
+            if len(self.rawdata) < n:
+                # Some data was parsed; resume on the next call.
+                self._parse_threshold = 1
+            else:
+                # Nothing was parsed; wait until the buffer doubles.
+                self._parse_threshold = len(self.rawdata)
 
     def close(self):
         """Handle any buffered data."""
+        if self._pending:
+            self.rawdata += ''.join(self._pending)
+            del self._pending[:]
+            self._pending_len = 0
         self.goahead(1)
 
     def error(self, message):

Lib/csv.py

@@ -214,12 +214,18 @@ def _guess_quote_and_delimiter(self, data, delimiters):
         this way.
         """
 
+        # The body of a quoted field ends at the first quote which is
+        # not doubled, as it does for a reader.  A lazy ".*?" scans to
+        # the end of the sample instead, from every start: quadratically.
+        # 2.7's re has no possessive "*+", so the body is matched
+        # atomically with a lookahead and a backreference.
+        body = r'(?=(?P<body>(?:(?P=quote){2}|(?!(?P=quote)).)*))(?P=body)'
         matches = []
-        for restr in ('(?P<delim>[^\w\n"\'])(?P<space> ?)(?P<quote>["\']).*?(?P=quote)(?P=delim)', # ,".*?",
-                      '(?:^|\n)(?P<quote>["\']).*?(?P=quote)(?P<delim>[^\w\n"\'])(?P<space> ?)',   #  ".*?",
-                      '(?P<delim>[^\w\n"\'])(?P<space> ?)(?P<quote>["\']).*?(?P=quote)(?:$|\n)',   # ,".*?"
-                      '(?:^|\n)(?P<quote>["\']).*?(?P=quote)(?:$|\n)'):                            #  ".*?" (no delim, no space)
-            regexp = re.compile(restr, re.DOTALL | re.MULTILINE)
+        for restr in (r'(?P<delim>[^\w\n"\'])(?P<space> ?)(?P<quote>["\'])%s(?P=quote)(?P=delim)',   # ,"...",
+                      r'(?:^|\n)(?P<quote>["\'])%s(?P=quote)(?P<delim>[^\w\n"\'])(?P<space> ?)',     #  "...",
+                      r'(?P<delim>[^\w\n"\'])(?P<space> ?)(?P<quote>["\'])%s(?P=quote)(?:$|\n)',  # ,"..."
+                      r'(?:^|\n)(?P<quote>["\'])%s(?P=quote)(?:$|\n)'):                           #  "..." (no delim, no space)
+            regexp = re.compile(restr % body, re.DOTALL | re.MULTILINE)
             matches = regexp.findall(data)
             if matches:
                 break

Lib/ftplib.py

@@ -939,7 +939,16 @@ def ftpcp(source, sourcename, target, targetname = '', type = 'I'):
     type = 'TYPE ' + type
     source.voidcmd(type)
     target.voidcmd(type)
-    sourcehost, sourceport = parse227(source.sendcmd('PASV'))
+    # Don't trust the IPv4 address the source server advertises in its PASV
+    # reply: a malicious source could otherwise point the target's data
+    # connection at an arbitrary host (SSRF).  A caller that needs the old
+    # behavior can set trust_server_pasv_ipv4_address on the source FTP
+    # object.  See FTP.makepasv(), which applies the same rule.
+    untrusted_host, sourceport = parse227(source.sendcmd('PASV'))
+    if source.trust_server_pasv_ipv4_address:
+        sourcehost = untrusted_host
+    else:
+        sourcehost = source.sock.getpeername()[0]
     target.sendport(sourcehost, sourceport)
     # RFC 959: the user must "listen" [...] BEFORE sending the
     # transfer request.

Lib/httplib.py

@@ -219,6 +219,13 @@
 # maximum amount of headers accepted
 _MAXHEADERS = 100
 
+# maximal number of interim (1xx) responses tolerated before the final
+# response.  Real servers send at most a few; without a bound, a server
+# streaming "100 Continue" responses would hang getresponse() forever.
+# A socket timeout cannot detect that as data keeps arriving within every
+# timeout window.
+_MAXINTERIMRESPONSES = 100
+
 # Header name/value ABNF (http://tools.ietf.org/html/rfc7230#section-3.2)
 #
 # VCHAR          = %x21-7E
@@ -261,6 +268,11 @@
 # to prevent http header injection.
 _contains_disallowed_method_pchar_re = re.compile('[\x00-\x1f]')
 
+# RFC 9112: Content-Length = 1*DIGIT and chunk-size = 1*HEXDIG.
+# int() is more permissive, so we match against the grammar before calling it.
+_is_legal_content_length = re.compile(r'[0-9]+\Z').match
+_is_legal_chunk_size = re.compile(r'[0-9a-fA-F]+\Z').match
+
 # We always set the Content-Length header for these methods because some
 # servers will otherwise respond with a 411
 _METHODS_EXPECTING_BODY = {'PATCH', 'POST', 'PUT'}
@@ -448,20 +460,27 @@ def begin(self):
             return
 
         # read until we get a non-100 response
-        while True:
+        for _ in range(_MAXINTERIMRESPONSES):
             version, status, reason = self._read_status()
             if status != CONTINUE:
                 break
             # skip the header from the 100 response
+            skipped = 0
             while True:
                 skip = self.fp.readline(_MAXLINE + 1)
                 if len(skip) > _MAXLINE:
                     raise LineTooLong("header line")
                 skip = skip.strip()
                 if not skip:
                     break
+                skipped += 1
+                if skipped > _MAXHEADERS:
+                    raise HTTPException("got more than %d headers" % _MAXHEADERS)
                 if self.debuglevel > 0:
                     print "header:", skip
+        else:
+            raise HTTPException("got more than %d interim responses"
+                                % _MAXINTERIMRESPONSES)
 
         self.status = status
         self.reason = reason.strip()
@@ -503,14 +522,12 @@ def begin(self):
         # do we have a Content-Length?
         # NOTE: RFC 2616, S4.4, #3 says we ignore this if tr_enc is "chunked"
         length = self.msg.getheader('content-length')
-        if length and not self.chunked:
-            try:
-                self.length = int(length)
-            except ValueError:
-                self.length = None
-            else:
-                if self.length < 0:  # ignore nonsensical negative lengths
-                    self.length = None
+        if length is not None:
+            # RFC 9112, section 5.1: optional whitespace (SP / HTAB) around
+            # the field value is not part of the value.
+            length = length.strip(' \t')
+        if length and not self.chunked and _is_legal_content_length(length):
+            self.length = int(length)
         else:
             self.length = None
 
@@ -632,7 +649,10 @@ def _read_chunked(self, amt):
                 i = line.find(';')
                 if i >= 0:
                     line = line[:i] # strip chunk-extensions
+                line = line.rstrip()
                 try:
+                    if not _is_legal_chunk_size(line):
+                        raise ValueError("invalid chunk size")
                     chunk_left = int(line, 16)
                 except ValueError:
                     # close the connection as protocol synchronisation is
@@ -662,6 +682,7 @@ def _read_chunked(self, amt):
 
         # read and discard trailer up to the CRLF terminator
         ### note: we shouldn't have any trailers!
+        trailers_read = 0
         while True:
             line = self.fp.readline(_MAXLINE + 1)
             if len(line) > _MAXLINE:
@@ -672,6 +693,13 @@ def _read_chunked(self, amt):
                 break
             if line == '\r\n':
                 break
+            # Bound the trailer count just as response headers are bounded.
+            # A server streaming trailer lines forever would otherwise hang
+            # the client; a socket timeout cannot detect that as data keeps
+            # arriving within every timeout window.
+            trailers_read += 1
+            if trailers_read > _MAXHEADERS:
+                raise HTTPException("got more than %d trailers" % _MAXHEADERS)
 
         # we read everything; close the "file"
         self.close()

Lib/mimetypes.py

@@ -37,17 +37,23 @@
     "add_type","read_mime_types","init"
 ]
 
-knownfiles = [
-    "/etc/mime.types",
-    "/etc/httpd/mime.types",                    # Mac OS X
-    "/etc/httpd/conf/mime.types",               # Apache
-    "/etc/apache/mime.types",                   # Apache 1
-    "/etc/apache2/mime.types",                  # Apache 2
-    "/usr/local/etc/httpd/conf/mime.types",
-    "/usr/local/lib/netscape/mime.types",
-    "/usr/local/etc/httpd/conf/mime.types",     # Apache 1.2
-    "/usr/local/etc/mime.types",                # Apache 1.3
-    ]
+if sys.platform == "win32":
+    # The default paths would resolve to the root of the current drive on Windows
+    # (e.g. C:\etc\mime.types), which unprivileged users can usually create.
+    # Only the registry is used by default.
+    knownfiles = []
+else:
+    knownfiles = [
+        "/etc/mime.types",
+        "/etc/httpd/mime.types",                    # Mac OS X
+        "/etc/httpd/conf/mime.types",               # Apache
+        "/etc/apache/mime.types",                   # Apache 1
+        "/etc/apache2/mime.types",                  # Apache 2
+        "/usr/local/etc/httpd/conf/mime.types",
+        "/usr/local/lib/netscape/mime.types",
+        "/usr/local/etc/httpd/conf/mime.types",     # Apache 1.2
+        "/usr/local/etc/mime.types",                # Apache 1.3
+        ]
 
 inited = False
 _db = None

Lib/tarfile.py

@@ -556,7 +556,9 @@ def seek(self, pos=0):
         if pos - self.pos >= 0:
             blocks, remainder = divmod(pos - self.pos, self.bufsize)
             for i in xrange(blocks):
-                self.read(self.bufsize)
+                data = self.read(self.bufsize)
+                if not data:
+                    break
             self.read(remainder)
         else:
             raise StreamError("seeking backwards is not allowed")

Lib/test/test_bz2.py

@@ -2,6 +2,7 @@
 from test.test_support import TESTFN, _4G, bigmemtest, import_module, findfile
 
 import unittest
+import binascii
 from cStringIO import StringIO
 import os
 import subprocess
@@ -44,6 +45,16 @@ def decompress(self, data):
             return bz2.decompress(data)
 
 
+# gh-150599: a crafted stream on which BZ2_bzDecompress() fails with
+# BZ_DATA_ERROR; calling it again on the same stream could write out of
+# bounds.
+DATA_ERROR_STREAM = binascii.unhexlify(
+    "425a6839314159265359000000000000007fffff000000000000000000000000"
+    "00000000000000000000000000000000000000e0370000000000000000000000"
+    "000000000000000000000000000000000000000000000000000083f3"
+)
+
+
 class BZ2FileTest(BaseTest):
     "Test BZ2File type miscellaneous methods."
 
@@ -337,6 +348,28 @@ def testInitNonExistentFile(self):
         self.assertRaises(IOError, f.__init__, "non-existent-file")
         del f
 
+    def test_read_after_data_error(self):
+        # gh-150599: BZ2File reads with BZ2_bzRead(), which re-enters
+        # BZ2_bzDecompress() on the same stream; after a data error the
+        # file refuses further reads until a seek() reopens the stream.
+        with open(self.filename, "wb") as f:
+            f.write(DATA_ERROR_STREAM + b"\x00" * 4000)
+        def readers(bz2f):
+            return [bz2f.read, lambda: bz2f.read(10), bz2f.readline,
+                    bz2f.readlines, lambda: next(iter(bz2f))]
+        for i in range(5):
+            with BZ2File(self.filename) as bz2f:
+                with self.assertRaisesRegexp(IOError, "invalid data stream"):
+                    bz2f.read()
+                with self.assertRaisesRegexp(ValueError, "previous error"):
+                    readers(bz2f)[i]()
+                # seek() starts over from a new stream, which fails again.
+                for offset, whence in ((0, 0), (10, 0), (0, 2)):
+                    with self.assertRaisesRegexp(IOError,
+                                                 "invalid data stream"):
+                        bz2f.seek(offset, whence)
+                        bz2f.read()
+
 class BZ2CompressorTest(BaseTest):
     def testCompress(self):
         # "Test BZ2Compressor.compress()/flush()"
@@ -417,6 +450,15 @@ def testEOFError(self):
         self.assertRaises(EOFError, bz2d.decompress, "anything")
         self.assertRaises(EOFError, bz2d.decompress, "")
 
+    def test_decompress_after_data_error(self):
+        # gh-150599: re-entering BZ2_bzDecompress() after an error could
+        # write out of bounds; the decompressor is now unusable instead.
+        bzd = BZ2Decompressor()
+        with self.assertRaisesRegexp(IOError, "invalid data stream"):
+            bzd.decompress(DATA_ERROR_STREAM)
+        with self.assertRaisesRegexp(ValueError, "previous error"):
+            bzd.decompress(b'\x00' * 18)
+
     @bigmemtest(_4G, memuse=1.25)
     def testBigmem(self, size):
         # Issue #14398: decompression fails when output data is >=2GB.

Lib/test/test_csv.py

@@ -1096,6 +1096,13 @@ def test_doublequote(self):
         dialect = sniffer.sniff(self.sample9)
         self.assertTrue(dialect.doublequote)
 
+    def test_sniff_quoted_single_column(self):
+        # gh-98820: this sample used to take minutes.
+        sniffer = csv.Sniffer()
+        sample = '"abcdefghijklmnopqrstuvwxyz"\n' * 10000
+        with self.assertRaisesRegexp(csv.Error, "Could not determine delimiter"):
+            sniffer.sniff(sample, delimiters=',:|\t')
+
 class NUL:
     def write(s, *args):
         pass