From 62b48bc8805fecf5772587fb69b889ebcdac3872 Mon Sep 17 00:00:00 2001 From: deadstrobe5 Date: Tue, 10 Nov 2026 03:57:06 +0000 Subject: [PATCH 1/2] Speed up documents with many lists --- CHANGES.md | 1 + lib/markdown2.py | 128 ++++++++++++++++++++-------------- test/test_redos.py | 6 ++ test/tm-cases/many_lists.html | 37 ++++++++++ test/tm-cases/many_lists.text | 25 +++++++ 5 files changed, 146 insertions(+), 51 deletions(-) create mode 100644 test/tm-cases/many_lists.html create mode 100644 test/tm-cases/many_lists.text diff --git a/CHANGES.md b/CHANGES.md index 10aabd33..d2ef21d3 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -2,6 +2,7 @@ ## python-markdown2 2.5.6 (not yet released) +- [pull #TBD] Speed up documents with many lists, which took quadratic time - [pull #762] Speed up documents with many links by skipping the backslash-escape scan for text without a backslash - [pull #748] Speed up `_unescape_special_chars` by skipping text with no hashes and dropping the full-text comparison per pass (#635) - [pull #749] URL-encode destinations generated by the `link-patterns` extra (#55) diff --git a/lib/markdown2.py b/lib/markdown2.py index d6285404..16b0ff7e 100755 --- a/lib/markdown2.py +++ b/lib/markdown2.py @@ -1858,61 +1858,87 @@ def _list_sub(self, match: re.Match[str]) -> str: def _do_lists(self, text: str) -> str: # Form HTML ordered (numbered) and unordered (bulleted) lists. - # Iterate over each *non-overlapping* list match. - pos = 0 - while True: - # Find the *first* hit for either list style (ul or ol). We - # match ul and ol separately to avoid adjacent lists of different - # types running into each other (see issue #16). - hits = [] - for marker_pat in (self._marker_ul, self._marker_ol): - less_than_tab = self.tab_width - 1 - other_marker_pat = self._marker_ul if marker_pat == self._marker_ol else self._marker_ol - whole_list = r''' - ( # \1 = whole list - ( # \2 - ([ ]{0,%d}) # \3 = the indentation level of the list item marker - (%s) # \4 = first list item marker - [ \t]+ - (?!\ *\4\ ) # '- - - ...' isn't a list. See 'not_quite_a_list' test case. + less_than_tab = self.tab_width - 1 + if self.list_level: # sub-list + prefix = "^" + else: + prefix = r"(?:(?<=\n\n)|\A\n?)" + # We match ul and ol separately to avoid adjacent lists of different + # types running into each other (see issue #16). + list_res = [] + bare_list_res = [] + for marker_pat, other_marker_pat in ((self._marker_ul, self._marker_ol), + (self._marker_ol, self._marker_ul)): + whole_list = r''' + ( # \1 = whole list + ( # \2 + ([ ]{0,%d}) # \3 = the indentation level of the list item marker + (%s) # \4 = first list item marker + [ \t]+ + (?!\ *\4\ ) # '- - - ...' isn't a list. See 'not_quite_a_list' test case. + ) + (?:.+?) + ( # \5 + \Z + | + \n{2,} + (?=\S) + (?! # Negative lookahead for another list item marker + [ \t]* + %s[ \t]+ ) - (?:.+?) - ( # \5 - \Z - | - \n{2,} - (?=\S) - (?! # Negative lookahead for another list item marker - [ \t]* - %s[ \t]+ - ) - | - \n+ - (?= - \3 # lookahead for a different style of list item marker - %s[ \t]+ - ) + | + \n+ + (?= + \3 # lookahead for a different style of list item marker + %s[ \t]+ ) - ) - ''' % (less_than_tab, marker_pat, marker_pat, other_marker_pat) - if self.list_level: # sub-list - list_re = re.compile("^"+whole_list, re.X | re.M | re.S) - else: - list_re = re.compile(r"(?:(?<=\n\n)|\A\n?)"+whole_list, - re.X | re.M | re.S) - match = list_re.search(text, pos) - if match: - hits.append((match.start(), match)) - if not hits: - break - hits.sort() - match = hits[0][1] + ) + ) + ''' % (less_than_tab, marker_pat, marker_pat, other_marker_pat) + list_res.append(re.compile(prefix + whole_list, re.X | re.M | re.S)) + bare_list_res.append(re.compile(whole_list, re.X | re.M | re.S)) + # Every list starts with an indented marker. Find the next marker + # first and try both list styles there, instead of searching the rest + # of the text for each style after every list. + list_start_re = re.compile(prefix + r"[ ]{0,%d}%s[ \t]+" % (less_than_tab, self._marker_any), + re.M) + + # Iterate over each *non-overlapping* list match. Converted lists are + # collected in `pieces` rather than spliced into `text`, which would + # copy the whole text once per list. + pieces = [] + pos = 0 + list_end = None + while True: + match = None + if pos == list_end: + # A converted list ends with a newline (a blank line at the + # top level), so another list may start right after it. + for list_re in bare_list_res: + match = list_re.match(text, pos) + if match: + break + if not match: + list_start = list_start_re.search(text, pos) + if not list_start: + break + for list_re in list_res: + match = list_re.match(text, list_start.start()) + if match: + break + if not match: + pos = list_start.start() + 1 + continue start, end = match.span() - middle = self._list_sub(match) - text = text[:start] + middle + text[end:] - pos = start + len(middle) # start pos for next attempted match + pieces.append(text[list_end or 0:start]) + pieces.append(self._list_sub(match)) + pos = list_end = end - return text + if not pieces: + return text + pieces.append(text[list_end:]) + return ''.join(pieces) _list_item_re = re.compile(r''' (\n)? # leading line = \1 diff --git a/test/test_redos.py b/test/test_redos.py index 1866a391..b575852e 100644 --- a/test/test_redos.py +++ b/test/test_redos.py @@ -66,6 +66,11 @@ def issue_metadata_list_redos(): return "---\nitems:\n - " + " " * 32000 + "\n---\n\n# hello\n" +def many_lists(): + # not technically a redos: each list used to rescan the rest of the text + return "".join(f"- item {i}\n\nParagraph {i}.\n\n" for i in range(3000)) + + # whack everything in a dict for easy lookup later on CASES = { fn.__name__: (fn, extras) @@ -82,6 +87,7 @@ def issue_metadata_list_redos(): (issue_707_spaced_attrs, None), (issue_metadata_redos, ['metadata']), (issue_metadata_list_redos, ['metadata']), + (many_lists, None), ] } diff --git a/test/tm-cases/many_lists.html b/test/tm-cases/many_lists.html new file mode 100644 index 00000000..0b0b6b08 --- /dev/null +++ b/test/tm-cases/many_lists.html @@ -0,0 +1,37 @@ + + +

Some text between lists.

+ + + +
+ + + +

I recommend upgrading to version +8. This line is not a list.

+ + + +
    +
  1. an ordered list
    2. +
  2. with two items
    3. +
+ +

Closing paragraph.

+ +
    +
  1. an ordered list
    2. +
  2. not starting at 1
    3. +
diff --git a/test/tm-cases/many_lists.text b/test/tm-cases/many_lists.text new file mode 100644 index 00000000..6441e565 --- /dev/null +++ b/test/tm-cases/many_lists.text @@ -0,0 +1,25 @@ +- first list at the start +- of the document + +Some text between lists. + +* second list +* with two items + +- - - + ++ third list, after a horizontal rule + +I recommend upgrading to version +8. This line is not a list. + +- an unordered list +- directly followed by + +1. an ordered list +2. with two items + +Closing paragraph. + +3. an ordered list +4. not starting at 1 From 253aad5f890efd10dbe62e618a7cbf84211e03aa Mon Sep 17 00:00:00 2001 From: deadstrobe5 Date: Tue, 10 Nov 2026 03:57:11 +0000 Subject: [PATCH 2/2] Add PR number to changelog --- CHANGES.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index d2ef21d3..35c06dd7 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -2,7 +2,7 @@ ## python-markdown2 2.5.6 (not yet released) -- [pull #TBD] Speed up documents with many lists, which took quadratic time +- [pull #772] Speed up documents with many lists, which took quadratic time - [pull #762] Speed up documents with many links by skipping the backslash-escape scan for text without a backslash - [pull #748] Speed up `_unescape_special_chars` by skipping text with no hashes and dropping the full-text comparison per pass (#635) - [pull #749] URL-encode destinations generated by the `link-patterns` extra (#55)