🐛 Use the CommonMark whitespace set, not the Python one

markdown_it/common/utils.py

@@ -191,6 +191,27 @@ def isWhiteSpace(code: int) -> bool:
     return code in MD_WHITESPACE
 
 
+#: The characters ``String.prototype.trim`` removes in JavaScript, which is
+#: what upstream markdown-it strips, minus U+FEFF.
+#:
+#: ``str.strip()`` without an argument uses :py:meth:`str.isspace`, which also
+#: removes U+001C, U+001D, U+001E, U+001F and U+0085. Those are not whitespace
+#: in CommonMark and are not removed by ``trim``, so relying on it drops them
+#: from the output and makes two different reference labels compare equal.
+#:
+#: U+FEFF is deliberately excluded: ``trim`` does remove it, and that has the
+#: very label folding effect this constant exists to avoid.
+MD_TRIM_CHARS = "".join(
+    chr(code)
+    for code in sorted(MD_WHITESPACE | set(range(0x2000, 0x200B)) | {0x2028, 0x2029})
+)
+
+
+def mdTrim(string: str) -> str:
+    """Strip leading and trailing whitespace, using the CommonMark set."""
+    return string.strip(MD_TRIM_CHARS)
+
+
 # //////////////////////////////////////////////////////////////////////////////
 
 
@@ -254,7 +275,7 @@ def normalizeReference(string: str) -> str:
     """Helper to unify [reference labels]."""
     # Trim and collapse whitespace
     #
-    string = re.sub(r"\s+", " ", string.strip())
+    string = re.sub("[" + re.escape(MD_TRIM_CHARS) + "]+", " ", mdTrim(string))
 
     # In node v10 'ẞ'.toLowerCase() === 'Ṿ', which is presumed to be a bug
     # fixed in v12 (couldn't find any details).

markdown_it/renderer.py

@@ -10,9 +10,10 @@ class Renderer
 
 from collections.abc import Sequence
 import inspect
+import re
 from typing import Any, ClassVar, Protocol
 
-from .common.utils import escapeHtml, unescapeAll
+from .common.utils import MD_TRIM_CHARS, escapeHtml, mdTrim, unescapeAll
 from .token import Token
 from .utils import EnvType, OptionsDict
 
@@ -266,12 +267,14 @@ def fence(
         env: EnvType,
     ) -> str:
         token = tokens[idx]
-        info = unescapeAll(token.info).strip() if token.info else ""
+        info = mdTrim(unescapeAll(token.info)) if token.info else ""
         langName = ""
         langAttrs = ""
 
         if info:
-            arr = info.split(maxsplit=1)
+            # Not ``str.split()``: it splits on the Python whitespace set,
+            # which is wider than the one upstream uses here.
+            arr = re.split("[" + re.escape(MD_TRIM_CHARS) + "]+", info, maxsplit=1)
             langName = arr[0]
             if len(arr) == 2:
                 langAttrs = arr[1]

markdown_it/rules_block/heading.py

@@ -4,7 +4,7 @@
 
 import logging
 
-from ..common.utils import isStrSpace
+from ..common.utils import isStrSpace, mdTrim
 from .state_block import StateBlock
 
 LOGGER = logging.getLogger(__name__)
@@ -59,7 +59,7 @@ def heading(state: StateBlock, startLine: int, endLine: int, silent: bool) -> bo
     token.map = [startLine, state.line]
 
     token = state.push("inline", "", 0)
-    token.content = state.src[pos:maximum].strip()
+    token.content = mdTrim(state.src[pos:maximum])
     token.map = [startLine, state.line]
     token.children = []
 

markdown_it/rules_block/lheading.py

@@ -1,6 +1,7 @@
 # lheading (---, ==)
 import logging
 
+from ..common.utils import mdTrim
 from .state_block import StateBlock
 
 LOGGER = logging.getLogger(__name__)
@@ -65,7 +66,7 @@ def lheading(state: StateBlock, startLine: int, endLine: int, silent: bool) -> b
         # Didn't find valid underline
         return False
 
-    content = state.getLines(startLine, nextLine, state.blkIndent, False).strip()
+    content = mdTrim(state.getLines(startLine, nextLine, state.blkIndent, False))
 
     state.line = nextLine + 1
 

markdown_it/rules_block/paragraph.py

@@ -2,6 +2,7 @@
 
 import logging
 
+from ..common.utils import mdTrim
 from .state_block import StateBlock
 
 LOGGER = logging.getLogger(__name__)
@@ -47,7 +48,7 @@ def paragraph(state: StateBlock, startLine: int, endLine: int, silent: bool) ->
 
         nextLine += 1
 
-    content = state.getLines(startLine, nextLine, state.blkIndent, False).strip()
+    content = mdTrim(state.getLines(startLine, nextLine, state.blkIndent, False))
 
     state.line = nextLine
 

markdown_it/rules_block/table.py

@@ -3,7 +3,7 @@
 
 import re
 
-from ..common.utils import charStrAt, isStrSpace
+from ..common.utils import charStrAt, isStrSpace, mdTrim
 from .state_block import StateBlock
 
 headerLineRe = re.compile(r"^:?-+:?$")
@@ -108,7 +108,7 @@ def table(state: StateBlock, startLine: int, endLine: int, silent: bool) -> bool
     columns = lineText.split("|")
     aligns = []
     for i in range(len(columns)):
-        t = columns[i].strip()
+        t = mdTrim(columns[i])
         if not t:
             # allow empty columns before and after table, but not in between columns;
             # e.g. allow ` |---| `, disallow ` ---||--- `
@@ -126,7 +126,7 @@ def table(state: StateBlock, startLine: int, endLine: int, silent: bool) -> bool
         else:
             aligns.append("")
 
-    lineText = getLine(state, startLine).strip()
+    lineText = mdTrim(getLine(state, startLine))
     if "|" not in lineText:
         return False
     if state.is_code_block(startLine):
@@ -171,7 +171,7 @@ def table(state: StateBlock, startLine: int, endLine: int, silent: bool) -> bool
         # note in markdown-it this map was removed in v12.0.0 however, we keep it,
         # since it is helpful to propagate to children tokens
         token.map = [startLine, startLine + 1]
-        token.content = columns[i].strip()
+        token.content = mdTrim(columns[i])
         token.children = []
 
         token = state.push("th_close", "th", -1)
@@ -193,7 +193,7 @@ def table(state: StateBlock, startLine: int, endLine: int, silent: bool) -> bool
 
         if terminate:
             break
-        lineText = getLine(state, nextLine).strip()
+        lineText = mdTrim(getLine(state, nextLine))
         if not lineText:
             break
         if state.is_code_block(nextLine):
@@ -227,7 +227,7 @@ def table(state: StateBlock, startLine: int, endLine: int, silent: bool) -> bool
             # since it is helpful to propagate to children tokens
             token.map = [nextLine, nextLine + 1]
             try:
-                token.content = columns[i].strip() if columns[i] else ""
+                token.content = mdTrim(columns[i]) if columns[i] else ""
             except IndexError:
                 token.content = ""
             token.children = []

tests/test_port/test_whitespace.py

@@ -0,0 +1,81 @@
+"""Whitespace handling must match upstream markdown-it, not Python's ``str``.
+
+``str.strip()`` and ``str.split()`` without arguments use :py:meth:`str.isspace`,
+which additionally treats U+001C, U+001D, U+001E, U+001F and U+0085 as
+whitespace. CommonMark and ``String.prototype.trim`` do not, so relying on them
+drops those characters from the output and folds two distinct reference labels
+into one.
+
+The expected values below are what ``markdown-it@14.1.0`` produces for the same
+input.
+"""
+
+import pytest
+
+from markdown_it import MarkdownIt
+
+#: Characters Python calls whitespace and CommonMark does not.
+EXTRA = ["\x1c", "\x1d", "\x1e", "\x1f", "\x85"]
+
+
+@pytest.mark.parametrize("char", EXTRA)
+def test_reference_label_is_not_folded(char):
+    """A definition whose label differs must not supply a different usage.
+
+    Before this was fixed, ``[a<char>b]`` and ``[a b]`` normalized to the same
+    label, so the definition resolved a usage that does not name it.
+    """
+    md = MarkdownIt("js-default")
+    assert md.render(f"[a b]\n\n[a{char}b]: http://example.com") == "<p>[a b]</p>\n"
+    assert (
+        md.render(f"[a{char}b]\n\n[a b]: http://example.com") == f"<p>[a{char}b]</p>\n"
+    )
+
+
+def test_reference_label_still_collapses_real_whitespace():
+    """Control: labels differing only in real whitespace still match."""
+    md = MarkdownIt("js-default")
+    assert (
+        md.render("[a b]\n\n[a\tb]: http://example.com")
+        == '<p><a href="http://example.com">a b</a></p>\n'
+    )
+
+
+@pytest.mark.parametrize("char", EXTRA)
+def test_character_survives_in_output(char):
+    """The character is content, so it has to reach the output."""
+    md = MarkdownIt("js-default")
+    assert md.render(f"x{char}") == f"<p>x{char}</p>\n"
+    assert md.render(f"{char}x") == f"<p>{char}x</p>\n"
+    assert md.render(f"# h{char}") == f"<h1>h{char}</h1>\n"
+    assert md.render(f"h{char}\n===") == f"<h1>h{char}</h1>\n"
+
+
+def test_real_whitespace_is_still_trimmed():
+    """Control: what CommonMark does call whitespace is still removed."""
+    md = MarkdownIt("js-default")
+    for char in ["\t", " ", "\xa0", " ", "　"]:
+        assert md.render(f"# h{char}") == "<h1>h</h1>\n", repr(char)
+
+
+def test_fence_info_is_not_split_on_it():
+    """``str.split()`` would split the info string on U+0085."""
+    md = MarkdownIt("js-default")
+    assert (
+        md.render("```py\x85rest\nx\n```")
+        == '<pre><code class="language-py\x85rest">x\n</code></pre>\n'
+    )
+
+
+def test_fence_info_still_splits_on_whitespace():
+    """Control: a real space still separates language from attributes."""
+    md = MarkdownIt("js-default")
+    assert (
+        md.render("```py extra\nx\n```")
+        == '<pre><code class="language-py">x\n</code></pre>\n'
+    )
+
+
+def test_table_cell_keeps_the_character():
+    md = MarkdownIt("js-default").enable("table")
+    assert "c\x85" in md.render("|a|\n|---|\n|c\x85|")