From 6f3b30ccb6d5b469d54802f6b91980fe485c35e2 Mon Sep 17 00:00:00 2001 From: Jared Lynskey Date: Thu, 15 Oct 2026 20:26:53 +0200 Subject: [PATCH] 0.3.0: html2markdown, HTML to CommonMark with GitHub's tables New html2markdown(html, *, base_url=None) (src/markdown.rs) walks the DOM html2text parses and writes CommonMark with GitHub's tables and strikethrough: inline links and images with absolute URLs, pipe tables, fenced code tagged with its language, `_em_` and `**strong**`, `-` bullets, and escapes only where Markdown would otherwise misread the text. Each paragraph with formatting is checked against the flanking rules and read back with pulldown-cmark; if it doesn't parse as intended it is rewritten with `*`, then without the formatting, so no stray delimiters reach the output. Checked by rendering the Markdown back with markdown-it and comparing words, links and images with the page (tests/markdown_check.py): 25 real pages, 172 Hugo pages and 50,000 random documents round-trip clean. markdownlint passes on 195 of 197 converted pages; the other two have lists the source numbers from 2, and one bold-italic construct that can only be written with both `_` and `*`. Hostile nesting and table spans stay linear. html2text() output is byte-identical to 0.2.0 (275 page/option hashes). Also: pin the fork commit that exports NodeData and resolve_url; tests and README; html2markdown in the PGO training and the benchmark; the round-trip tests' dependencies in CI. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01WobasfsvfjJy3c6nkovhov --- .github/workflows/CI.yml | 12 +- CHANGELOG.md | 10 + Cargo.lock | 29 +- Cargo.toml | 12 +- README.md | 86 +- bench/bench.py | 36 +- pgo/train.py | 10 +- pyproject.toml | 2 +- python/pyhtml2text/__init__.py | 4 +- python/pyhtml2text/__init__.pyi | 1 + src/lib.rs | 87 +- src/markdown.rs | 2442 +++++++++++++++++++++++++++++++ tests/markdown_check.py | 464 ++++++ tests/test_markdown.py | 207 +++ 14 files changed, 3334 insertions(+), 68 deletions(-) create mode 100644 src/markdown.rs create mode 100644 tests/markdown_check.py create mode 100644 tests/test_markdown.py diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 446a350..e6e541f 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -78,7 +78,7 @@ jobs: uv venv .venv source .venv/bin/activate uv pip install pyhtml2text --no-index --no-deps --find-links dist --reinstall - uv pip install pyhtml2text pytest + uv pip install pyhtml2text pytest markdown-it-py html5lib pytest - name: pytest if: ${{ contains(fromJSON('["armv7", "s390x", "ppc64le"]'), matrix.platform.target) }} @@ -94,7 +94,7 @@ jobs: set -e python3 -m venv .venv source .venv/bin/activate - pip install pytest + pip install pytest markdown-it-py html5lib pip install pyhtml2text --no-index --no-deps --find-links dist --force-reinstall pip install pyhtml2text pytest @@ -146,7 +146,7 @@ jobs: python3 -m virtualenv .venv source .venv/bin/activate pip install pyhtml2text --no-index --no-deps --find-links dist --force-reinstall - pip install pyhtml2text pytest + pip install pyhtml2text pytest markdown-it-py html5lib pytest ' - name: pytest @@ -163,7 +163,7 @@ jobs: python3 -m virtualenv .venv source .venv/bin/activate pip install pyhtml2text --no-index --no-deps --find-links dist --force-reinstall - pip install pyhtml2text pytest + pip install pyhtml2text pytest markdown-it-py html5lib pytest windows: @@ -224,7 +224,7 @@ jobs: uv venv .venv source .venv/Scripts/activate uv pip install pyhtml2text --no-index --no-deps --find-links dist --reinstall - uv pip install pyhtml2text pytest + uv pip install pyhtml2text pytest markdown-it-py html5lib pytest macos: @@ -278,7 +278,7 @@ jobs: uv venv .venv source .venv/bin/activate uv pip install pyhtml2text --no-index --no-deps --find-links dist --reinstall - uv pip install pyhtml2text pytest + uv pip install pyhtml2text pytest markdown-it-py html5lib pytest sdist: diff --git a/CHANGELOG.md b/CHANGELOG.md index a08d69b..989982d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,15 @@ # Changelog +## 0.3.0 (unreleased) + +- New `html2markdown(html, *, base_url=None)`: HTML to CommonMark with + GitHub's tables and strikethrough. Links and images keep their URLs (made + absolute with `base_url`), tables become pipe tables, code blocks are + fenced with their language, and characters are escaped only where Markdown + would misread them. 3–13× faster than markdownify, html2text and + html-to-markdown on real pages. +- `html2text()` output is unchanged. + ## 0.2.0 (2026-10-15) - 1.5–1.7× faster than 0.1.0 on real web pages, and 2× faster on short diff --git a/Cargo.lock b/Cargo.lock index e6a17eb..56c994a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -51,7 +51,7 @@ checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" [[package]] name = "html2text" version = "0.17.1" -source = "git+https://github.com/jaredlynskey/rust-html2text?rev=9c109fac38728040cf3c32b411305cdd840bd511#9c109fac38728040cf3c32b411305cdd840bd511" +source = "git+https://github.com/jaredlynskey/rust-html2text?rev=b42930f8e6688ea121daf392e27a7d9812e6192d#b42930f8e6688ea121daf392e27a7d9812e6192d" dependencies = [ "html5ever", "tendril", @@ -117,6 +117,12 @@ dependencies = [ "web_atoms", ] +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + [[package]] name = "memoffset" version = "0.9.1" @@ -230,12 +236,25 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "pulldown-cmark" +version = "0.13.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e9f068eba8e7071c5f9511831b44f32c740d5adf574e990f946ddb53db2f314e" +dependencies = [ + "bitflags", + "memchr", + "unicase", +] + [[package]] name = "pyhtml2text" -version = "0.2.0" +version = "0.3.0" dependencies = [ "html2text", + "html5ever", "mimalloc", + "pulldown-cmark", "pyo3", ] @@ -458,6 +477,12 @@ dependencies = [ "syn 3.0.9", ] +[[package]] +name = "unicase" +version = "2.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "357cc3acc6a036009fd6c973ed009037c732d60d0b4f6c673e9041497482a28f" + [[package]] name = "unicode-ident" version = "1.0.27" diff --git a/Cargo.toml b/Cargo.toml index d654561..9a0259f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pyhtml2text" -version = "0.2.0" +version = "0.3.0" edition = "2021" license = "MIT" publish = false @@ -11,9 +11,13 @@ crate-type = ["cdylib"] [dependencies] pyo3 = { version = "0.26", features = ["extension-module", "abi3-py310"] } -# Fork of jugglerchris/rust-html2text with Config::base_url and some -# performance changes (branch pyhtml2text-perf). -html2text = { git = "https://github.com/jaredlynskey/rust-html2text", rev = "9c109fac38728040cf3c32b411305cdd840bd511" } +# Fork of jugglerchris/rust-html2text with Config::base_url, some performance +# changes and the DOM exports the Markdown output uses (branch pyhtml2text-perf). +html2text = { git = "https://github.com/jaredlynskey/rust-html2text", rev = "b42930f8e6688ea121daf392e27a7d9812e6192d" } +# The parser html2text uses; patched below. Used directly to read element attributes. +html5ever = "0.39" +# Reads the Markdown back to check that emphasis parses as intended. +pulldown-cmark = { version = "0.13", default-features = false } # Much faster than the system allocators for html2text's many small allocations. # Not on glibc armv7/ppc64le/s390x: their manylinux2014 cross compilers are too diff --git a/README.md b/README.md index b173708..10ce69b 100644 --- a/README.md +++ b/README.md @@ -1,10 +1,59 @@ # pyhtml2text -Fast HTML to plain text for Python, built on +Fast HTML to plain text and to Markdown for Python, built on [rust-html2text](https://github.com/jugglerchris/rust-html2text). -It adds one thing the upstream crate doesn't have yet: relative links can be -resolved against the page's URL, so footnote links are usable on their own. +Relative links can be resolved against the page's URL, so the output's links +work on their own. + +## Markdown + +```python +from pyhtml2text import html2markdown + +html = """

Before you start

+

Read the setup guide and back up your data.

+How it fits together +
PlanPrice
BasicFree
""" + +print(html2markdown(html, base_url="https://example.com/docs/intro.html")) +``` + +```markdown +## Before you start + +Read the [setup guide](https://example.com/docs/setup.html) and **back up** your data. + +![How it fits together](https://example.com/img/diagram.png) + +| Plan | Price | +| --- | --- | +| Basic | Free | +``` + +`html2markdown(html, *, base_url=None)` writes +[CommonMark](https://commonmark.org/) with GitHub's tables and +strikethrough, in the usual conventions: + +- `#` headings, `-` bullets, `_emphasis_` and `**strong**`, `~~strikethrough~~`. +- Inline links and images, `[text](url)` and `![alt](src)`, made absolute when + `base_url` is given. A link whose only content is an icon uses its + `aria-label` or `title` as its text. +- Fenced code blocks, tagged with the language when the page names one + (`class="language-python"`). +- Pipe tables. Cells spanning rows or columns become empty cells, a table + without a header row gets an empty one, and tables used for page layout are + written as plain paragraphs. +- Characters are escaped only where Markdown would otherwise read them as + syntax, so `snake_case`, `5 * 3` and `$5` stay as they are. +- Lines aren't wrapped. Hidden content (`hidden`, `aria-hidden`, + `display: none`), scripts, styles and form controls are left out. + +The output is checked by rendering it back with a CommonMark parser and +comparing every word, link and image with the original page, on real sites +and on tens of thousands of random documents (`tests/markdown_check.py`). + +## Plain text ```python from pyhtml2text import html2text @@ -17,11 +66,6 @@ print(html2text(html, base_url="https://example.com/docs/page.html")) # [1]: https://example.com/login ``` -`html` can be a `str` or UTF-8 `bytes` (e.g. `response.content`). Requires -Python 3.10 or later. - -## Options - `html2text(html, width=80, *, ...)` | option | default | | @@ -34,9 +78,21 @@ Python 3.10 or later. | `max_wrap_width` | `None` | wrap paragraphs narrower than `width` | | `raw` | `False` | flatten tables so cell text stays together | +Both functions take a `str` or UTF-8 `bytes` (e.g. `response.content`). +Requires Python 3.10 or later. + ## Performance -Median time to render a page, on an Apple Silicon Mac: +Median time on an Apple Silicon Mac. To Markdown: + +| | Wikipedia article (1 MB) | BBC News front page (370 KB) | Python docs page (245 KB) | +|---|---|---|---| +| [markdownify](https://pypi.org/project/markdownify/) | 184 ms | 28 ms | 97 ms | +| [html2text](https://pypi.org/project/html2text/) (pure Python) | 73 ms | 13 ms | 37 ms | +| [html-to-markdown](https://pypi.org/project/html-to-markdown/) | 44 ms | 5.6 ms | 15 ms | +| **pyhtml2text** | **14 ms** | **1.7 ms** | **4.5 ms** | + +To plain text: | | Wikipedia article (780 KB) | BBC News front page (370 KB) | |---|---|---| @@ -44,7 +100,7 @@ Median time to render a page, on an Apple Silicon Mac: | [html2text-rs](https://pypi.org/project/html2text-rs/) | 15.2 ms | 2.5 ms | | **pyhtml2text** | **10.9 ms** | **1.6 ms** | -- The GIL is released while rendering, so threads scale across cores. On +- The GIL is released while converting, so threads scale across cores. On free-threaded Python (3.14t) the module runs without re-enabling the GIL. - Wheels for Linux (x86_64, aarch64), macOS and Windows x64 are built with profile-guided optimisation, which makes them about 25% faster. Other @@ -54,9 +110,11 @@ Median time to render a page, on an Apple Silicon Mac: ## Credits -The rendering is done by rust-html2text by Chris Emerson (MIT). This package -uses [a fork](https://github.com/jaredlynskey/rust-html2text/tree/pyhtml2text-perf) +The plain-text rendering is done by rust-html2text by Chris Emerson (MIT). +This package uses [a fork](https://github.com/jaredlynskey/rust-html2text/tree/pyhtml2text-perf) that adds `base_url` and some performance changes, and a patched copy of [html5ever](https://github.com/servo/html5ever) (MIT/Apache-2.0, see -`vendor/`). It isn't affiliated with or endorsed by either upstream project, so -please report issues here rather than upstream. +`vendor/`). Markdown output is checked with +[pulldown-cmark](https://github.com/raphlinus/pulldown-cmark) (MIT). It +isn't affiliated with or endorsed by any of these projects, so please report +issues here rather than upstream. diff --git a/bench/bench.py b/bench/bench.py index b4c2a8c..6b62a5c 100644 --- a/bench/bench.py +++ b/bench/bench.py @@ -15,6 +15,7 @@ import sys import time +import pyhtml2text from pyhtml2text import html2text BASE = "https://example.com/dir/page.html" @@ -66,26 +67,45 @@ def run(html, opts): return f"ValueError: {e}" +def run_markdown(html, base_url=None): + try: + return pyhtml2text.html2markdown(html, base_url=base_url) + except (AttributeError, ValueError) as e: # AttributeError: before 0.3.0 + return f"{type(e).__name__}: {e}" + + def hashes(corpus): out = {} for name, html in inputs(corpus): for oname, opts in OPTION_SETS.items(): out[f"{name}/{oname}"] = hashlib.sha256(run(html, opts).encode()).hexdigest() + if hasattr(pyhtml2text, "html2markdown"): + for oname, base in (("markdown", None), ("markdown_base_url", BASE)): + digest = hashlib.sha256(run_markdown(html, base).encode()).hexdigest() + out[f"{name}/{oname}"] = digest return out +def median_ms(fn, html, repeat): + fn(html) + xs = [] + for _ in range(repeat): + t = time.perf_counter() + fn(html) + xs.append(time.perf_counter() - t) + return statistics.median(xs) * 1000 + + def timings(corpus, repeat): - print(f"{'input':22s} {'KiB':>6s} {'median ms':>10s}") + markdown = hasattr(pyhtml2text, "html2markdown") + print(f"{'input':22s} {'KiB':>6s} {'text ms':>10s}" + (f" {'markdown ms':>12s}" if markdown else "")) for name, html in inputs(corpus): if name in EDGE_CASES: continue - run(html, {}) - xs = [] - for _ in range(repeat): - t = time.perf_counter() - run(html, {}) - xs.append(time.perf_counter() - t) - print(f"{name:22s} {len(html.encode()) / 1024:6.0f} {statistics.median(xs) * 1000:10.2f}") + line = f"{name:22s} {len(html.encode()) / 1024:6.0f} {median_ms(lambda h: run(h, {}), html, repeat):10.2f}" + if markdown: + line += f" {median_ms(run_markdown, html, repeat):12.2f}" + print(line) tiny = "

Hello world, link

" n = 20000 t = time.perf_counter() diff --git a/pgo/train.py b/pgo/train.py index 5b148a9..0b8a9db 100644 --- a/pgo/train.py +++ b/pgo/train.py @@ -10,7 +10,7 @@ import pathlib import sys -from pyhtml2text import html2text +from pyhtml2text import html2markdown, html2text CORPUS = pathlib.Path(__file__).parent / "corpus" OPTION_SETS = [ @@ -42,11 +42,15 @@ def main(): for opts in OPTION_SETS: html2text(text, **opts) html2text(page) + html2markdown(text) + html2markdown(page, base_url="https://example.com/docs/page.html") for _ in range(5000): for snippet in SNIPPETS: html2text(snippet) - html2text("