Speed up documents with many lists

CHANGES.md

@@ -2,6 +2,7 @@
 
 ## python-markdown2 2.5.6 (not yet released)
 
+- [pull #772] Speed up documents with many lists, which took quadratic time
 - [pull #762] Speed up documents with many links by skipping the backslash-escape scan for text without a backslash
 - [pull #748] Speed up `_unescape_special_chars` by skipping text with no hashes and dropping the full-text comparison per pass (#635)
 - [pull #749] URL-encode destinations generated by the `link-patterns` extra (#55)

lib/markdown2.py

@@ -1858,61 +1858,87 @@ def _list_sub(self, match: re.Match[str]) -> str:
     def _do_lists(self, text: str) -> str:
         # Form HTML ordered (numbered) and unordered (bulleted) lists.
 
-        # Iterate over each *non-overlapping* list match.
-        pos = 0
-        while True:
-            # Find the *first* hit for either list style (ul or ol). We
-            # match ul and ol separately to avoid adjacent lists of different
-            # types running into each other (see issue #16).
-            hits = []
-            for marker_pat in (self._marker_ul, self._marker_ol):
-                less_than_tab = self.tab_width - 1
-                other_marker_pat = self._marker_ul if marker_pat == self._marker_ol else self._marker_ol
-                whole_list = r'''
-                    (                   # \1 = whole list
-                      (                 # \2
-                        ([ ]{0,%d})     # \3 = the indentation level of the list item marker
-                        (%s)            # \4 = first list item marker
-                        [ \t]+
-                        (?!\ *\4\ )     # '- - - ...' isn't a list. See 'not_quite_a_list' test case.
+        less_than_tab = self.tab_width - 1
+        if self.list_level:  # sub-list
+            prefix = "^"
+        else:
+            prefix = r"(?:(?<=\n\n)|\A\n?)"
+        # We match ul and ol separately to avoid adjacent lists of different
+        # types running into each other (see issue #16).
+        list_res = []
+        bare_list_res = []
+        for marker_pat, other_marker_pat in ((self._marker_ul, self._marker_ol),
+                                             (self._marker_ol, self._marker_ul)):
+            whole_list = r'''
+                (                   # \1 = whole list
+                  (                 # \2
+                    ([ ]{0,%d})     # \3 = the indentation level of the list item marker
+                    (%s)            # \4 = first list item marker
+                    [ \t]+
+                    (?!\ *\4\ )     # '- - - ...' isn't a list. See 'not_quite_a_list' test case.
+                  )
+                  (?:.+?)
+                  (                 # \5
+                      \Z
+                    |
+                      \n{2,}
+                      (?=\S)
+                      (?!           # Negative lookahead for another list item marker
+                        [ \t]*
+                        %s[ \t]+
                       )
-                      (?:.+?)
-                      (                 # \5
-                          \Z
-                        |
-                          \n{2,}
-                          (?=\S)
-                          (?!           # Negative lookahead for another list item marker
-                            [ \t]*
-                            %s[ \t]+
-                          )
-                        |
-                          \n+
-                          (?=
-                            \3          # lookahead for a different style of list item marker
-                            %s[ \t]+
-                          )
+                    |
+                      \n+
+                      (?=
+                        \3          # lookahead for a different style of list item marker
+                        %s[ \t]+
                       )
-                    )
-                ''' % (less_than_tab, marker_pat, marker_pat, other_marker_pat)
-                if self.list_level:  # sub-list
-                    list_re = re.compile("^"+whole_list, re.X | re.M | re.S)
-                else:
-                    list_re = re.compile(r"(?:(?<=\n\n)|\A\n?)"+whole_list,
-                                         re.X | re.M | re.S)
-                match = list_re.search(text, pos)
-                if match:
-                    hits.append((match.start(), match))
-            if not hits:
-                break
-            hits.sort()
-            match = hits[0][1]
+                  )
+                )
+            ''' % (less_than_tab, marker_pat, marker_pat, other_marker_pat)
+            list_res.append(re.compile(prefix + whole_list, re.X | re.M | re.S))
+            bare_list_res.append(re.compile(whole_list, re.X | re.M | re.S))
+        # Every list starts with an indented marker. Find the next marker
+        # first and try both list styles there, instead of searching the rest
+        # of the text for each style after every list.
+        list_start_re = re.compile(prefix + r"[ ]{0,%d}%s[ \t]+" % (less_than_tab, self._marker_any),
+                                   re.M)
+
+        # Iterate over each *non-overlapping* list match. Converted lists are
+        # collected in `pieces` rather than spliced into `text`, which would
+        # copy the whole text once per list.
+        pieces = []
+        pos = 0
+        list_end = None
+        while True:
+            match = None
+            if pos == list_end:
+                # A converted list ends with a newline (a blank line at the
+                # top level), so another list may start right after it.
+                for list_re in bare_list_res:
+                    match = list_re.match(text, pos)
+                    if match:
+                        break
+            if not match:
+                list_start = list_start_re.search(text, pos)
+                if not list_start:
+                    break
+                for list_re in list_res:
+                    match = list_re.match(text, list_start.start())
+                    if match:
+                        break
+                if not match:
+                    pos = list_start.start() + 1
+                    continue
             start, end = match.span()
-            middle = self._list_sub(match)
-            text = text[:start] + middle + text[end:]
-            pos = start + len(middle)  # start pos for next attempted match
+            pieces.append(text[list_end or 0:start])
+            pieces.append(self._list_sub(match))
+            pos = list_end = end
 
-        return text
+        if not pieces:
+            return text
+        pieces.append(text[list_end:])
+        return ''.join(pieces)
 
     _list_item_re = re.compile(r'''
         (\n)?                   # leading line = \1

test/test_redos.py

@@ -66,6 +66,11 @@ def issue_metadata_list_redos():
     return "---\nitems:\n  - " + " " * 32000 + "\n---\n\n# hello\n"
 
 
+def many_lists():
+    # not technically a redos: each list used to rescan the rest of the text
+    return "".join(f"- item {i}\n\nParagraph {i}.\n\n" for i in range(3000))
+
+
 # whack everything in a dict for easy lookup later on
 CASES = {
     fn.__name__: (fn, extras)
@@ -82,6 +87,7 @@ def issue_metadata_list_redos():
         (issue_707_spaced_attrs, None),
         (issue_metadata_redos, ['metadata']),
         (issue_metadata_list_redos, ['metadata']),
+        (many_lists, None),
     ]
 }
 

test/tm-cases/many_lists.html

@@ -0,0 +1,37 @@
+<ul>
+<li>first list at the start</li>
+<li>of the document</li>
+</ul>
+
+<p>Some text between lists.</p>
+
+<ul>
+<li>second list</li>
+<li>with two items</li>
+</ul>
+
+<hr />
+
+<ul>
+<li>third list, after a horizontal rule</li>
+</ul>
+
+<p>I recommend upgrading to version
+8. This line is not a list.</p>
+
+<ul>
+<li>an unordered list</li>
+<li>directly followed by</li>
+</ul>
+
+<ol>
+<li>an ordered list</li>
+<li>with two items</li>
+</ol>
+
+<p>Closing paragraph.</p>
+
+<ol start="3">
+<li>an ordered list</li>
+<li>not starting at 1</li>
+</ol>

test/tm-cases/many_lists.text

@@ -0,0 +1,25 @@
+- first list at the start
+- of the document
+
+Some text between lists.
+
+* second list
+* with two items
+
+- - -
+
++ third list, after a horizontal rule
+
+I recommend upgrading to version
+8. This line is not a list.
+
+- an unordered list
+- directly followed by
+
+1. an ordered list
+2. with two items
+
+Closing paragraph.
+
+3. an ordered list
+4. not starting at 1