0.3.0: html2markdown (HTML to Markdown)

.github/workflows/CI.yml

@@ -78,7 +78,7 @@ jobs:
           uv venv .venv
           source .venv/bin/activate
           uv pip install pyhtml2text --no-index --no-deps --find-links dist --reinstall
-          uv pip install pyhtml2text pytest
+          uv pip install pyhtml2text pytest markdown-it-py html5lib
           pytest
       - name: pytest
         if: ${{ contains(fromJSON('["armv7", "s390x", "ppc64le"]'), matrix.platform.target) }}
@@ -94,7 +94,7 @@ jobs:
             set -e
             python3 -m venv .venv
             source .venv/bin/activate
-            pip install pytest
+            pip install pytest markdown-it-py html5lib
             pip install pyhtml2text --no-index --no-deps --find-links dist --force-reinstall
             pip install pyhtml2text
             pytest
@@ -146,7 +146,7 @@ jobs:
             python3 -m virtualenv .venv
             source .venv/bin/activate
             pip install pyhtml2text --no-index --no-deps --find-links dist --force-reinstall
-            pip install pyhtml2text pytest
+            pip install pyhtml2text pytest markdown-it-py html5lib
             pytest
           '
       - name: pytest
@@ -163,7 +163,7 @@ jobs:
             python3 -m virtualenv .venv
             source .venv/bin/activate
             pip install pyhtml2text --no-index --no-deps --find-links dist --force-reinstall
-            pip install pyhtml2text pytest
+            pip install pyhtml2text pytest markdown-it-py html5lib
             pytest
 
   windows:
@@ -224,7 +224,7 @@ jobs:
           uv venv .venv
           source .venv/Scripts/activate
           uv pip install pyhtml2text --no-index --no-deps --find-links dist --reinstall
-          uv pip install pyhtml2text pytest
+          uv pip install pyhtml2text pytest markdown-it-py html5lib
           pytest
 
   macos:
@@ -278,7 +278,7 @@ jobs:
           uv venv .venv
           source .venv/bin/activate
           uv pip install pyhtml2text --no-index --no-deps --find-links dist --reinstall
-          uv pip install pyhtml2text pytest
+          uv pip install pyhtml2text pytest markdown-it-py html5lib
           pytest
 
   sdist:

CHANGELOG.md

@@ -1,5 +1,15 @@
 # Changelog
 
+## 0.3.0 (unreleased)
+
+- New `html2markdown(html, *, base_url=None)`: HTML to CommonMark with
+  GitHub's tables and strikethrough. Links and images keep their URLs (made
+  absolute with `base_url`), tables become pipe tables, code blocks are
+  fenced with their language, and characters are escaped only where Markdown
+  would misread them. 3–13× faster than markdownify, html2text and
+  html-to-markdown on real pages.
+- `html2text()` output is unchanged.
+
 ## 0.2.0 (2026-10-15)
 
 - 1.5–1.7× faster than 0.1.0 on real web pages, and 2× faster on short

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "pyhtml2text"
-version = "0.2.0"
+version = "0.3.0"
 edition = "2021"
 license = "MIT"
 publish = false
@@ -11,9 +11,13 @@ crate-type = ["cdylib"]
 
 [dependencies]
 pyo3 = { version = "0.26", features = ["extension-module", "abi3-py310"] }
-# Fork of jugglerchris/rust-html2text with Config::base_url and some
-# performance changes (branch pyhtml2text-perf).
-html2text = { git = "https://github.com/jaredlynskey/rust-html2text", rev = "9c109fac38728040cf3c32b411305cdd840bd511" }
+# Fork of jugglerchris/rust-html2text with Config::base_url, some performance
+# changes and the DOM exports the Markdown output uses (branch pyhtml2text-perf).
+html2text = { git = "https://github.com/jaredlynskey/rust-html2text", rev = "b42930f8e6688ea121daf392e27a7d9812e6192d" }
+# The parser html2text uses; patched below. Used directly to read element attributes.
+html5ever = "0.39"
+# Reads the Markdown back to check that emphasis parses as intended.
+pulldown-cmark = { version = "0.13", default-features = false }
 
 # Much faster than the system allocators for html2text's many small allocations.
 # Not on glibc armv7/ppc64le/s390x: their manylinux2014 cross compilers are too

README.md

@@ -1,10 +1,59 @@
 # pyhtml2text
 
-Fast HTML to plain text for Python, built on
+Fast HTML to plain text and to Markdown for Python, built on
 [rust-html2text](https://github.com/jugglerchris/rust-html2text).
 
-It adds one thing the upstream crate doesn't have yet: relative links can be
-resolved against the page's URL, so footnote links are usable on their own.
+Relative links can be resolved against the page's URL, so the output's links
+work on their own.
+
+## Markdown
+
+```python
+from pyhtml2text import html2markdown
+
+html = """<h2>Before you start</h2>
+<p>Read the <a href="setup.html">setup guide</a> and <b>back up</b> your data.</p>
+<img src="/img/diagram.png" alt="How it fits together">
+<table><tr><th>Plan</th><th>Price</th></tr><tr><td>Basic</td><td>Free</td></tr></table>"""
+
+print(html2markdown(html, base_url="https://example.com/docs/intro.html"))
+```
+
+```markdown
+## Before you start
+
+Read the [setup guide](https://example.com/docs/setup.html) and **back up** your data.
+
+![How it fits together](https://example.com/img/diagram.png)
+
+| Plan | Price |
+| --- | --- |
+| Basic | Free |
+```
+
+`html2markdown(html, *, base_url=None)` writes
+[CommonMark](https://commonmark.org/) with GitHub's tables and
+strikethrough, in the usual conventions:
+
+- `#` headings, `-` bullets, `_emphasis_` and `**strong**`, `~~strikethrough~~`.
+- Inline links and images, `[text](url)` and `![alt](src)`, made absolute when
+  `base_url` is given. A link whose only content is an icon uses its
+  `aria-label` or `title` as its text.
+- Fenced code blocks, tagged with the language when the page names one
+  (`class="language-python"`).
+- Pipe tables. Cells spanning rows or columns become empty cells, a table
+  without a header row gets an empty one, and tables used for page layout are
+  written as plain paragraphs.
+- Characters are escaped only where Markdown would otherwise read them as
+  syntax, so `snake_case`, `5 * 3` and `$5` stay as they are.
+- Lines aren't wrapped. Hidden content (`hidden`, `aria-hidden`,
+  `display: none`), scripts, styles and form controls are left out.
+
+The output is checked by rendering it back with a CommonMark parser and
+comparing every word, link and image with the original page, on real sites
+and on tens of thousands of random documents (`tests/markdown_check.py`).
+
+## Plain text
 
 ```python
 from pyhtml2text import html2text
@@ -17,11 +66,6 @@ print(html2text(html, base_url="https://example.com/docs/page.html"))
 # [1]: https://example.com/login
 ```
 
-`html` can be a `str` or UTF-8 `bytes` (e.g. `response.content`). Requires
-Python 3.10 or later.
-
-## Options
-
 `html2text(html, width=80, *, ...)`
 
 | option | default | |
@@ -34,17 +78,29 @@ Python 3.10 or later.
 | `max_wrap_width` | `None` | wrap paragraphs narrower than `width` |
 | `raw` | `False` | flatten tables so cell text stays together |
 
+Both functions take a `str` or UTF-8 `bytes` (e.g. `response.content`).
+Requires Python 3.10 or later.
+
 ## Performance
 
-Median time to render a page, on an Apple Silicon Mac:
+Median time on an Apple Silicon Mac. To Markdown:
+
+| | Wikipedia article (1 MB) | BBC News front page (370 KB) | Python docs page (245 KB) |
+|---|---|---|---|
+| [markdownify](https://pypi.org/project/markdownify/) | 184 ms | 28 ms | 97 ms |
+| [html2text](https://pypi.org/project/html2text/) (pure Python) | 73 ms | 13 ms | 37 ms |
+| [html-to-markdown](https://pypi.org/project/html-to-markdown/) | 44 ms | 5.6 ms | 15 ms |
+| **pyhtml2text** | **14 ms** | **1.7 ms** | **4.5 ms** |
+
+To plain text:
 
 | | Wikipedia article (780 KB) | BBC News front page (370 KB) |
 |---|---|---|
 | [html2text](https://pypi.org/project/html2text/) (pure Python) | 71.5 ms | 14.9 ms |
 | [html2text-rs](https://pypi.org/project/html2text-rs/) | 15.2 ms | 2.5 ms |
 | **pyhtml2text** | **10.9 ms** | **1.6 ms** |
 
-- The GIL is released while rendering, so threads scale across cores. On
+- The GIL is released while converting, so threads scale across cores. On
   free-threaded Python (3.14t) the module runs without re-enabling the GIL.
 - Wheels for Linux (x86_64, aarch64), macOS and Windows x64 are built with
   profile-guided optimisation, which makes them about 25% faster. Other
@@ -54,9 +110,11 @@ Median time to render a page, on an Apple Silicon Mac:
 
 ## Credits
 
-The rendering is done by rust-html2text by Chris Emerson (MIT). This package
-uses [a fork](https://github.com/jaredlynskey/rust-html2text/tree/pyhtml2text-perf)
+The plain-text rendering is done by rust-html2text by Chris Emerson (MIT).
+This package uses [a fork](https://github.com/jaredlynskey/rust-html2text/tree/pyhtml2text-perf)
 that adds `base_url` and some performance changes, and a patched copy of
 [html5ever](https://github.com/servo/html5ever) (MIT/Apache-2.0, see
-`vendor/`). It isn't affiliated with or endorsed by either upstream project, so
-please report issues here rather than upstream.
+`vendor/`). Markdown output is checked with
+[pulldown-cmark](https://github.com/raphlinus/pulldown-cmark) (MIT). It
+isn't affiliated with or endorsed by any of these projects, so please report
+issues here rather than upstream.

bench/bench.py

@@ -15,6 +15,7 @@
 import sys
 import time
 
+import pyhtml2text
 from pyhtml2text import html2text
 
 BASE = "https://example.com/dir/page.html"
@@ -66,26 +67,45 @@ def run(html, opts):
         return f"ValueError: {e}"
 
 
+def run_markdown(html, base_url=None):
+    try:
+        return pyhtml2text.html2markdown(html, base_url=base_url)
+    except (AttributeError, ValueError) as e:  # AttributeError: before 0.3.0
+        return f"{type(e).__name__}: {e}"
+
+
 def hashes(corpus):
     out = {}
     for name, html in inputs(corpus):
         for oname, opts in OPTION_SETS.items():
             out[f"{name}/{oname}"] = hashlib.sha256(run(html, opts).encode()).hexdigest()
+        if hasattr(pyhtml2text, "html2markdown"):
+            for oname, base in (("markdown", None), ("markdown_base_url", BASE)):
+                digest = hashlib.sha256(run_markdown(html, base).encode()).hexdigest()
+                out[f"{name}/{oname}"] = digest
     return out
 
 
+def median_ms(fn, html, repeat):
+    fn(html)
+    xs = []
+    for _ in range(repeat):
+        t = time.perf_counter()
+        fn(html)
+        xs.append(time.perf_counter() - t)
+    return statistics.median(xs) * 1000
+
+
 def timings(corpus, repeat):
-    print(f"{'input':22s} {'KiB':>6s} {'median ms':>10s}")
+    markdown = hasattr(pyhtml2text, "html2markdown")
+    print(f"{'input':22s} {'KiB':>6s} {'text ms':>10s}" + (f" {'markdown ms':>12s}" if markdown else ""))
     for name, html in inputs(corpus):
         if name in EDGE_CASES:
             continue
-        run(html, {})
-        xs = []
-        for _ in range(repeat):
-            t = time.perf_counter()
-            run(html, {})
-            xs.append(time.perf_counter() - t)
-        print(f"{name:22s} {len(html.encode()) / 1024:6.0f} {statistics.median(xs) * 1000:10.2f}")
+        line = f"{name:22s} {len(html.encode()) / 1024:6.0f} {median_ms(lambda h: run(h, {}), html, repeat):10.2f}"
+        if markdown:
+            line += f" {median_ms(run_markdown, html, repeat):12.2f}"
+        print(line)
     tiny = "<p>Hello <b>world</b>, <a href='/x'>link</a></p>"
     n = 20000
     t = time.perf_counter()

pgo/train.py

@@ -10,7 +10,7 @@
 import pathlib
 import sys
 
-from pyhtml2text import html2text
+from pyhtml2text import html2markdown, html2text
 
 CORPUS = pathlib.Path(__file__).parent / "corpus"
 OPTION_SETS = [
@@ -42,11 +42,15 @@ def main():
             for opts in OPTION_SETS:
                 html2text(text, **opts)
             html2text(page)
+            html2markdown(text)
+            html2markdown(page, base_url="https://example.com/docs/page.html")
     for _ in range(5000):
         for snippet in SNIPPETS:
             html2text(snippet)
-    html2text("<ul><li>" * 2000 + "x")
-    html2text("<blockquote>" * 2000 + "x")
+            html2markdown(snippet)
+    for nested in ("<ul><li>" * 2000 + "x", "<blockquote>" * 2000 + "x"):
+        html2text(nested)
+        html2markdown(nested)
     print(f"trained on {len(pages)} pages x {len(OPTION_SETS) + 1} option sets x {rounds} rounds")
 
 

pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "maturin"
 
 [project]
 name = "pyhtml2text"
-version = "0.2.0"
+version = "0.3.0"
 description = "Fast HTML to plain text, with relative link resolution (Python bindings to rust-html2text)"
 readme = "README.md"
 license = "MIT"

python/pyhtml2text/__init__.py

@@ -1,3 +1,3 @@
-from ._pyhtml2text import html2text
+from ._pyhtml2text import html2markdown, html2text
 
-__all__ = ["html2text"]
+__all__ = ["html2markdown", "html2text"]

python/pyhtml2text/__init__.pyi

@@ -10,3 +10,4 @@ def html2text(
     max_wrap_width: int | None = None,
     raw: bool = False,
 ) -> str: ...
+def html2markdown(html: str | bytes, *, base_url: str | None = None) -> str: ...