diff --git a/.github/pytest.sh b/.github/pytest.sh new file mode 100755 index 0000000..6a18b07 --- /dev/null +++ b/.github/pytest.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash +# Installs the wheel built into dist/ into a fresh virtualenv for each Python +# version given (or the default Python, with none) and runs the tests. +set -euo pipefail + +# Only uv's own Pythons: otherwise a free-threaded 3.15t that the runner has +# for building would do for "3.15". +export UV_PYTHON_PREFERENCE=only-managed + +for py in "${@:-}"; do + rm -rf .venv-test + uv venv -q ${py:+-p "$py"} .venv-test + if [ -d .venv-test/Scripts ]; then python=.venv-test/Scripts/python.exe; else python=.venv-test/bin/python; fi + uv pip install -q -p "$python" pyhtml2text --no-index --no-deps --find-links dist --reinstall + uv pip install -q -p "$python" pyhtml2text pytest markdown-it-py html5lib + "$python" -c 'import sys; print(sys.version)' + "$python" -m pytest +done diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index f6fceb6..595f079 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -59,7 +59,7 @@ jobs: uses: PyO3/maturin-action@v1 with: target: ${{ matrix.platform.target }} - args: --release --out dist -i python3.14t + args: --release --out dist -i python3.14t python3.15t sccache: ${{ !startsWith(github.ref, 'refs/tags/') }} manylinux: auto rust-toolchain: ${{ env.PGO_RUST_TOOLCHAIN }} @@ -68,18 +68,11 @@ jobs: with: name: wheels-linux-${{ matrix.platform.target }} path: dist - - uses: astral-sh/setup-uv@v10.3.2 + - uses: astral-sh/setup-uv@v10.3.3 if: ${{ matrix.platform.target == 'x86_64' || matrix.platform.target == 'aarch64' }} - name: pytest if: ${{ matrix.platform.target == 'x86_64' || matrix.platform.target == 'aarch64' }} - shell: bash - run: | - set -e - uv venv .venv - source .venv/bin/activate - uv pip install pyhtml2text --no-index --no-deps --find-links dist --reinstall - uv pip install pyhtml2text pytest markdown-it-py html5lib - pytest + run: .github/pytest.sh 3.10 3.14t 3.15 3.15t - name: pytest if: ${{ contains(fromJSON('["armv7", "s390x", "ppc64le"]'), matrix.platform.target) }} uses: uraimo/run-on-arch-action@v3 @@ -129,7 +122,7 @@ jobs: uses: PyO3/maturin-action@v1 with: target: ${{ matrix.platform.target }} - args: --release --out dist -i python3.14t + args: --release --out dist -i python3.14t python3.15t sccache: ${{ !startsWith(github.ref, 'refs/tags/') }} manylinux: musllinux_1_2 - name: Upload wheels @@ -176,12 +169,18 @@ jobs: target: x64 python_arch: x64 pgo: true + pythons: 3.10 3.14t 3.15 3.15t - runner: windows-latest target: x86 python_arch: x86 + pythons: >- + cpython-3.10-windows-x86 cpython-3.14t-windows-x86 + cpython-3.15-windows-x86 cpython-3.15t-windows-x86 - runner: windows-11-arm target: aarch64 python_arch: arm64 + # There is no uv-managed Python 3.10 for arm64 Windows. + pythons: 3.11 3.14t 3.15 3.15t steps: - uses: actions/checkout@v7 - uses: actions/setup-python@v7 @@ -198,34 +197,34 @@ jobs: with: target: ${{ matrix.platform.target }} args: --release --out dist - sccache: ${{ !startsWith(github.ref, 'refs/tags/') }} + # sccache before 0.19 can't parse the `-Tp ` that cc 1.9 and + # later pass cl.exe for mimalloc's C++ source (rust-lang/cc-rs#2063). + sccache: false rust-toolchain: ${{ env.PGO_RUST_TOOLCHAIN }} - uses: actions/setup-python@v7 with: - python-version: "3.14t" + python-version: | + 3.14t + 3.15t architecture: ${{ matrix.platform.python_arch }} - name: Build free-threaded wheels uses: PyO3/maturin-action@v1 with: target: ${{ matrix.platform.target }} - args: --release --out dist -i python3.14t - sccache: ${{ !startsWith(github.ref, 'refs/tags/') }} + args: --release --out dist -i python3.14t python3.15t + # sccache before 0.19 can't parse the `-Tp ` that cc 1.9 and + # later pass cl.exe for mimalloc's C++ source (rust-lang/cc-rs#2063). + sccache: false rust-toolchain: ${{ env.PGO_RUST_TOOLCHAIN }} - name: Upload wheels uses: actions/upload-artifact@v7 with: name: wheels-windows-${{ matrix.platform.target }} path: dist - - uses: astral-sh/setup-uv@v10.3.2 + - uses: astral-sh/setup-uv@v10.3.3 - name: pytest shell: bash - run: | - set -e - uv venv .venv - source .venv/Scripts/activate - uv pip install pyhtml2text --no-index --no-deps --find-links dist --reinstall - uv pip install pyhtml2text pytest markdown-it-py html5lib - pytest + run: .github/pytest.sh ${{ matrix.platform.pythons }} macos: runs-on: ${{ matrix.platform.runner }} @@ -258,12 +257,14 @@ jobs: rust-toolchain: ${{ env.PGO_RUST_TOOLCHAIN }} - uses: actions/setup-python@v7 with: - python-version: "3.14t" + python-version: | + 3.14t + 3.15t - name: Build free-threaded wheels uses: PyO3/maturin-action@v1 with: target: ${{ matrix.platform.target }} - args: --release --out dist -i python3.14t + args: --release --out dist -i python3.14t python3.15t sccache: ${{ !startsWith(github.ref, 'refs/tags/') }} rust-toolchain: ${{ env.PGO_RUST_TOOLCHAIN }} - name: Upload wheels @@ -271,15 +272,9 @@ jobs: with: name: wheels-macos-${{ matrix.platform.target }} path: dist - - uses: astral-sh/setup-uv@v10.3.2 + - uses: astral-sh/setup-uv@v10.3.3 - name: pytest - run: | - set -e - uv venv .venv - source .venv/bin/activate - uv pip install pyhtml2text --no-index --no-deps --find-links dist --reinstall - uv pip install pyhtml2text pytest markdown-it-py html5lib - pytest + run: .github/pytest.sh 3.10 3.14t 3.15 3.15t sdist: runs-on: ubuntu-latest @@ -305,10 +300,9 @@ jobs: name: pypi url: https://pypi.org/p/pyhtml2text permissions: - # Use to sign the release artifacts + # Trusted publishing to PyPI, and signing the attestations id-token: write - # Used to upload release artifacts - contents: write + contents: read # Used to generate artifact attestation attestations: write steps: @@ -319,7 +313,7 @@ jobs: subject-path: 'wheels-*/*' - name: Install uv if: ${{ startsWith(github.ref, 'refs/tags/') }} - uses: astral-sh/setup-uv@v10.3.2 + uses: astral-sh/setup-uv@v10.3.3 - name: Publish to PyPI if: ${{ startsWith(github.ref, 'refs/tags/') }} # Trusted publishing (OIDC) - no token stored in the repo. diff --git a/.gitignore b/.gitignore index 5e9a49e..45ef673 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,6 @@ target/ dist/ -.venv/ +.venv*/ __pycache__/ *.so .DS_Store diff --git a/CHANGELOG.md b/CHANGELOG.md index 5bad363..6e8a2ac 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,23 @@ # Changelog +## Unreleased + +- Deeply nested lists, definitions, paragraphs and emphasis are rendered in + linear time again. In 0.4.0 each level cost time for every level around + it: 400,000 nested lists took 49 s, and now take 0.5 s. Output is + unchanged. +- Wheels for free-threaded Python 3.15 (3.15t), alongside 3.14t. The abi3 + wheels already cover Python 3.15. CI now tests on 3.10, 3.14t, 3.15 and + 3.15t. +- Character widths from Unicode 18 (unicode-width 0.2.3). The halfwidth + katakana sound marks ゙ and ゚ count one column, as terminals show them, + rather than none, and characters new in Unicode 18 (emoji, kana, + ideographic characters and combining marks) get their own widths. Other + text wraps as before. +- html5ever 0.40.1, still with + [servo/html5ever#800](https://github.com/servo/html5ever/pull/800). + Output is unchanged. Building from source needs Rust 1.85 or later. + ## 0.4.0 (2026-10-22) - `html2text()` is rendered by this package's own code. The library it used diff --git a/Cargo.lock b/Cargo.lock index 2f6c40c..fa50ce9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -10,9 +10,9 @@ checksum = "3ded4057c258ba199e2d26386d3af3780957ecaee6c4ef4041c6b4b8b97c0b06" [[package]] name = "cc" -version = "1.9.0" +version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "877bf67ac8e52449eb2f9e98a344ea163beb70dd1f9eebe3cc5357dcfeaee4df" +checksum = "86e055fa6e3e40ad6187ba25b6855eeb635d5271f8e0f0e1e9a5826b8d75bea0" dependencies = [ "find-msvc-tools", "shlex", @@ -44,10 +44,11 @@ checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" [[package]] name = "html5ever" -version = "0.39.0" +version = "0.40.1" dependencies = [ "log", "markup5ever", + "memchr", ] [[package]] @@ -82,9 +83,9 @@ checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6" [[package]] name = "markup5ever" -version = "0.39.0" +version = "0.40.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7122d987ec5f704ee56f6e5b41a7d93722e9aae27ae07cafa4036c4d3f9757de" +checksum = "0ab3dc68ac4a0f5719e560136778c1ee716e296030d75dbd4484e37e39e3a842" dependencies = [ "log", "tendril", @@ -143,9 +144,9 @@ dependencies = [ [[package]] name = "phf" -version = "0.13.1" +version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1562dc717473dbaa4c1f85a36410e03c047b2e7df7f45ee938fbef64ae7fadf" +checksum = "010378780309880b08997fae13be7834dba947d36393bd372f2b1556deb2a2f6" dependencies = [ "phf_shared", "serde", @@ -153,9 +154,9 @@ dependencies = [ [[package]] name = "phf_codegen" -version = "0.13.1" +version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49aa7f9d80421bca176ca8dbfebe668cc7a2684708594ec9f3c0db0805d5d6e1" +checksum = "41b585a510fb76fdebead6897982ef2a03a21d8e6cbcca904999742a4afc6ffe" dependencies = [ "phf_generator", "phf_shared", @@ -163,9 +164,9 @@ dependencies = [ [[package]] name = "phf_generator" -version = "0.13.1" +version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "135ace3a761e564ec88c03a77317a7c6b80bb7f7135ef2544dbe054243b89737" +checksum = "aeb62e0959d5a1bebc965f4d15d9e2b7cea002b6b0f5ba8cde6cc26738467100" dependencies = [ "fastrand", "phf_shared", @@ -173,9 +174,9 @@ dependencies = [ [[package]] name = "phf_shared" -version = "0.13.1" +version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e57fef6bc5981e38c2ce2d63bfa546861309f875b8a75f092d1d54ae2d64f266" +checksum = "c6fd9027e2d9319be6349febd1db4e8d02aa544921200c9b777720ac34a3aa89" dependencies = [ "siphasher", ] @@ -347,15 +348,15 @@ checksum = "33f4fe9184a62d842c9ef383018f3306d8ba224fd9d836f56d7288308847c256" [[package]] name = "smallvec" -version = "1.16.3" +version = "1.16.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b3dc8af474f516a851ff4bd12db780f948b9250ad37211e4eec0bccea54e01b" +checksum = "d0d5d4b1981b8bbbb7aed204cae5726d683d5d6ac7e74ec54122ea28f315e7c8" [[package]] name = "string_cache" -version = "0.9.0" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a18596f8c785a729f2819c0f6a7eae6ebeebdfffbfe4214ae6b087f690e31901" +checksum = "ffa8a5dbe8b3f0bbe29d4c3225daafaeead63afdc1b65fc4c01a1384166038e6" dependencies = [ "new_debug_unreachable", "parking_lot", @@ -365,9 +366,9 @@ dependencies = [ [[package]] name = "string_cache_codegen" -version = "0.6.1" +version = "0.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "585635e46db231059f76c5849798146164652513eb9e8ab2685939dd90f29b69" +checksum = "928dcdf75e47626b3617a976ec205d9f057584c371c1f23b782129268d0e6edc" dependencies = [ "phf_generator", "phf_shared", @@ -415,15 +416,15 @@ checksum = "a2c754d6c33795a1c324727428e5a7dedb5b06195f9890bdbcba760d3e246563" [[package]] name = "unicode-width" -version = "0.2.2" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4ac048d71ede7ee76d585517add45da530660ef4390e49b098733c6e897f254" +checksum = "ca718471876992059f251e33aafc334346ada08042b7661dd1fe313a9bced557" [[package]] name = "web_atoms" -version = "0.2.6" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba8b815c1b593dc0baf78dd0f4fc8fdb2de53198fb1163738093e9a311c33fb3" +checksum = "7572660c8890448ba236b7376f27e389c6a7e1c70195622faced601f855c0ada" dependencies = [ "phf", "phf_codegen", diff --git a/Cargo.toml b/Cargo.toml index c062393..2a74e55 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -12,9 +12,10 @@ crate-type = ["cdylib"] [dependencies] pyo3 = { version = "0.30", features = ["abi3-py310"] } # The HTML parser; patched below. -html5ever = "0.39" -# Display width of characters, for wrapping. -unicode-width = "=0.2.2" +html5ever = "0.40" +# Display width of characters, for wrapping. Pinned, since newer Unicode data +# can change the output. +unicode-width = "=0.2.3" # Reads the Markdown back to check that emphasis parses as intended. pulldown-cmark = { version = "0.13", default-features = false } diff --git a/README.md b/README.md index 61f8fa2..6e2b9cc 100644 --- a/README.md +++ b/README.md @@ -18,6 +18,8 @@ html = """

Before you start

print(html2markdown(html, base_url="https://example.com/docs/intro.html")) ``` +Output: + ```markdown ## Before you start @@ -100,12 +102,13 @@ To plain text: | **pyhtml2text** | **7.0 ms** | **1.3 ms** | - The GIL is released while converting, so threads scale across cores. On - free-threaded Python (3.14t) the module runs without re-enabling the GIL. + free-threaded Python (3.14t and 3.15t) the module runs without re-enabling + the GIL. - Wheels for Linux (x86_64, aarch64), macOS and Windows x64 are built with profile-guided optimisation, which makes them about 25% faster. Other platforms get the same code without it. - Deeply nested HTML is parsed in linear time, so hostile input can't stall a - worker: 40,000 nested lists take 0.14 s, against 6 s with stock html5ever, + worker: 40,000 nested lists take 0.02 s, against 18 s with stock html5ever, and 100,000 levels of nesting followed by unmatched end tags take under 0.1 s. diff --git a/pyproject.toml b/pyproject.toml index 197e484..6d0ff4f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -15,7 +15,17 @@ keywords = ["html", "text", "markdown", "html2text"] classifiers = [ "Programming Language :: Rust", "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3 :: Only", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3.14", + "Programming Language :: Python :: 3.15", + "Programming Language :: Python :: Free Threading :: 3 - Stable", "Topic :: Text Processing :: Markup :: HTML", + "Topic :: Text Processing :: Markup :: Markdown", + "Typing :: Typed", ] [project.urls] diff --git a/src/table.rs b/src/table.rs index d3718b9..dc311ff 100644 --- a/src/table.rs +++ b/src/table.rs @@ -756,11 +756,7 @@ fn layout(r: &mut Renderer, cells: &mut [Cell], nrows_all: usize, width: usize) fn vertical(cells: &[Cell], nrows: usize, width: usize, borders: bool) -> Job { let mut pieces = Vec::new(); let mut to_render = Vec::new(); - let rule = || Line { - text: "─".repeat(width), - width, - kind: LineKind::Rule, - }; + let rule = || Line::of_kind("─".repeat(width), width, LineKind::Rule); let sep = || Line::new("/".repeat(width), width); if borders { pieces.push(Some(rule())); @@ -1020,15 +1016,12 @@ impl Grid { } fn row_line(&self, row: usize, line: usize, sep: &str) -> Line { - let mut out = Line { - text: String::new(), - width: 0, - kind: if sep == " " { - LineKind::Text - } else { - LineKind::Row - }, + let kind = if sep == " " { + LineKind::Text + } else { + LineKind::Row }; + let mut out = Line::of_kind(String::new(), 0, kind); for (i, &(idx, _, _)) in self.vis[row].iter().enumerate() { if i > 0 { out.text.push_str(sep); @@ -1134,11 +1127,7 @@ impl Grid { } } // Lay the spanning cells' text over the line. - let mut out = Line { - text: String::new(), - width: 0, - kind: LineKind::Rule, - }; + let mut out = Line::of_kind(String::new(), 0, LineKind::Rule); texts.sort_by_key(|t| t.0); let mut ti = texts.into_iter().peekable(); let mut x = 0; diff --git a/src/text.rs b/src/text.rs index 79fe3ce..dd7c3d4 100644 --- a/src/text.rs +++ b/src/text.rs @@ -7,6 +7,7 @@ //! pending and only carried out when something is written after them, so //! empty blocks leave no trace and adjacent requests merge. +use std::collections::HashMap; use std::mem; use unicode_width::{UnicodeWidthChar, UnicodeWidthStr}; @@ -85,7 +86,11 @@ pub fn render(dom: &Dom, opts: &Options) -> Result { parent.r.footnotes = mem::take(&mut done.r.footnotes); parent.r.table_sizes = mem::take(&mut done.r.table_sizes); let lines = done.r.bufs.pop().unwrap().finish(); - parent.table.as_mut().unwrap().add(lines); + parent + .table + .as_mut() + .unwrap() + .add(lines.into_iter().map(Line::flatten).collect()); } Ok(levels.pop().unwrap().r.finish()) } @@ -113,6 +118,10 @@ pub struct Line { pub text: String, pub width: usize, pub kind: LineKind, + /// The prefixes of the blocks the line is in, still to be put in front + /// of `text`, written back to front: each block then adds its own in + /// time for its length, however deeply blocks are nested. + rprefix: String, } /// What a line is, for a table it ends up in. @@ -127,16 +136,40 @@ pub enum LineKind { impl Line { pub fn new(text: String, width: usize) -> Line { + Line::of_kind(text, width, LineKind::Text) + } + + pub fn of_kind(text: String, width: usize, kind: LineKind) -> Line { Line { text, width, - kind: LineKind::Text, + kind, + rprefix: String::new(), } } fn empty() -> Line { Line::new(String::new(), 0) } + + fn add_prefix(&mut self, prefix: &str) { + self.rprefix.extend(prefix.chars().rev()); + self.width += prefix.len(); + // A prefix makes a table's lines plain text. + self.kind = LineKind::Text; + } + + /// The line with its prefixes in front of it. + fn flatten(mut self) -> Line { + if !self.rprefix.is_empty() { + let mut text = String::with_capacity(self.rprefix.len() + self.text.len()); + text.extend(self.rprefix.chars().rev()); + text.push_str(&self.text); + self.text = text; + self.rprefix.clear(); + } + self + } } /// One buffer of output lines. @@ -150,7 +183,7 @@ struct Buf { word: String, word_w: usize, pieces: Vec, - word_style: Vec, + word_style: u32, /// The word has a character of non-zero width. word_base: bool, /// A space is due before the next word. @@ -188,7 +221,7 @@ impl Buf { word: String::new(), word_w: 0, pieces: Vec::new(), - word_style: Vec::new(), + word_style: 0, word_base: false, space: false, open: false, @@ -410,8 +443,11 @@ pub struct Renderer<'a> { /// Sizes of the tables seen so far, shared with the cells' renderers. pub table_sizes: table::Sizes, /// The styled elements (emphasis, code, links...) the text is inside. - /// Text in one style is measured as one piece. - style: Vec, + /// Text in one style is measured as one piece. Each distinct stack of + /// them is numbered (`styles`, 0 for none), and this holds the number of + /// the stack up to each of them. + style: Vec, + styles: HashMap<(u32, u8), u32>, /// Width of the leading whitespace of a `
` line, or of whitespace
     /// after a tab, held until something follows it.
     prews: usize,
@@ -427,6 +463,7 @@ impl<'a> Renderer<'a> {
             footnotes: Vec::new(),
             table_sizes: table::Sizes::new(),
             style: Vec::new(),
+            styles: HashMap::new(),
             prews: 0,
             after_tab: false,
         }
@@ -437,11 +474,26 @@ impl<'a> Renderer<'a> {
     }
 
     fn mark(&mut self) {
-        for f in &mut self.frames {
+        // Frames are marked from the outermost in, so once one is found
+        // marked, so are all those outside it.
+        for f in self.frames.iter_mut().rev() {
+            if f.wrote {
+                break;
+            }
             f.wrote = true;
         }
     }
 
+    fn push_style(&mut self, id: u8) {
+        let next = self.styles.len() as u32 + 1;
+        let style = *self.styles.entry((self.cur_style(), id)).or_insert(next);
+        self.style.push(style);
+    }
+
+    fn cur_style(&self) -> u32 {
+        self.style.last().copied().unwrap_or(0)
+    }
+
     /// The first thing happens in a buffer: carry out the separator before
     /// its block, in the parent (and so on up).
     fn wake(&mut self) {
@@ -558,10 +610,11 @@ impl<'a> Renderer<'a> {
 
     fn word_char(&mut self, c: char) {
         let w = char_width(c);
+        let style = self.cur_style();
         let b = self.bufs.last_mut().unwrap();
-        if b.word_style != self.style {
+        if b.word_style != style {
             b.piece();
-            b.word_style.clone_from(&self.style);
+            b.word_style = style;
         }
         b.word.push(c);
         b.word_w += w;
@@ -599,6 +652,7 @@ impl<'a> Renderer<'a> {
                 self.touch();
             }
         }
+        let style = self.cur_style();
         for c in seg.chars() {
             let b = self.bufs.last_mut().unwrap();
             let at_start = (b.cur.is_empty() || self.after_tab) && b.word.is_empty();
@@ -641,9 +695,9 @@ impl<'a> Renderer<'a> {
                 }
                 b.chop(&" ".repeat(n));
             }
-            if b.word_style != self.style {
+            if b.word_style != style {
                 b.piece();
-                b.word_style.clone_from(&self.style);
+                b.word_style = style;
             }
             b.word.push(c);
             b.word_w += char_width(c);
@@ -731,22 +785,12 @@ impl<'a> Renderer<'a> {
         let lines = self.bufs.pop().unwrap().finish();
         if !lines.is_empty() {
             let b = self.b();
-            for (i, line) in lines.into_iter().enumerate() {
+            for (i, mut line) in lines.into_iter().enumerate() {
                 let prefix = if i == 0 { first } else { rest };
-                let mut s = String::with_capacity(prefix.len() + line.text.len());
-                s.push_str(prefix);
-                s.push_str(&line.text);
-                // A prefix makes a table's lines plain text.
-                let kind = if prefix.is_empty() {
-                    line.kind
-                } else {
-                    LineKind::Text
-                };
-                b.lines.push(Line {
-                    text: s,
-                    width: prefix.len() + line.width,
-                    kind,
-                });
+                if !prefix.is_empty() {
+                    line.add_prefix(prefix);
+                }
+                b.lines.push(line);
             }
             b.blank_after = false;
             b.content = true;
@@ -833,7 +877,7 @@ impl<'a> Renderer<'a> {
         self.start(Kind::Lead(LINE));
         stack.push(Step::Leave(Leave::Frame));
         // Unlike emphasis, the markers are part of the term's style.
-        self.style.push(b'd');
+        self.push_style(b'd');
         stack.push(Step::Leave(Leave::Styled("")));
         if self.opts.decorate {
             self.text("*");
@@ -938,7 +982,7 @@ impl<'a> Renderer<'a> {
                     "code" => {
                         // Unlike emphasis, the markers are part of the code's
                         // style.
-                        self.style.push(b'c');
+                        self.push_style(b'c');
                         if decorate {
                             self.text("`");
                             stack.push(Step::Leave(Leave::Styled("")));
@@ -953,7 +997,7 @@ impl<'a> Renderer<'a> {
                     "ins" => ("", "", b'n'),
                     _ if sup_digits(node).is_some() => {
                         let digits = sup_digits(node).unwrap();
-                        self.style.push(b's');
+                        self.push_style(b's');
                         self.text(&digits);
                         self.style.pop();
                         return;
@@ -963,12 +1007,12 @@ impl<'a> Renderer<'a> {
                 if !open.is_empty() {
                     self.text(open);
                 }
-                self.style.push(id);
+                self.push_style(id);
                 Leave::Styled(close)
             }
             "del" | "s" => {
                 self.b().strike += 1;
-                self.style.push(b'x');
+                self.push_style(b'x');
                 Leave::Strike
             }
             "a" => match attr("href") {
@@ -982,7 +1026,7 @@ impl<'a> Renderer<'a> {
                         None => href,
                     };
                     self.text("[");
-                    self.style.push(b'a');
+                    self.push_style(b'a');
                     Leave::Link(url)
                 }
                 None => Leave::Nothing,
@@ -1179,6 +1223,7 @@ impl<'a> Renderer<'a> {
         let lines = b.finish();
         let mut out = String::new();
         for line in lines {
+            out.extend(line.rprefix.chars().rev());
             out.push_str(&line.text);
             out.push('\n');
         }
diff --git a/tests/test_basic.py b/tests/test_basic.py
index 05ca950..d58a999 100644
--- a/tests/test_basic.py
+++ b/tests/test_basic.py
@@ -97,6 +97,30 @@ def test_deep_nesting_is_linear():
             assert elapsed < 10, f"took {elapsed:.1f}s on {html[:20]!r}..."
 
 
+def test_deep_nesting_renders_in_linear_time():
+    # In 0.4.0 each level of these cost time for every level around it: the
+    # lists took 49s and 28s, the others 6s. Now each takes under a second.
+    import platform
+    import sys
+    import time
+
+    n = 400_000
+    cases = [
+        ("
  • " * n + "x", "* " * n + "x\n"), + ("
    " * n + "x", " " * n + "x\n"), + ("

    a" * (n // 2), "\n\n".join(["a"] * (n // 2)) + "\n"), + # Wrapped at 80 columns. + ("" * n + "x", "**" * n + "x" + "**" * n), + ] + for html, expected in cases: + start = time.perf_counter() + out = html2text(html) + elapsed = time.perf_counter() - start + assert (out if expected.endswith("\n") else out.replace("\n", "")) == expected + if platform.machine().lower() in ("x86_64", "amd64") or sys.platform == "darwin": + assert elapsed < 10, f"took {elapsed:.1f}s on {html[:20]!r}..." + + def test_deeply_nested_tables(): # Each table's cells are rendered on their own; nesting them mustn't # take a call stack as deep as the tables (or time quadratic in it). diff --git a/vendor/README.md b/vendor/README.md index 29be8c0..772a5a0 100644 --- a/vendor/README.md +++ b/vendor/README.md @@ -1,6 +1,6 @@ # Vendored crates -## html5ever 0.39.0 +## html5ever 0.40.1 Copied from the crates.io release (`src/`, licences, README and manifest; examples, tests and benchmarks removed). Dual-licensed MIT / Apache-2.0, @@ -9,9 +9,9 @@ see `html5ever/LICENSE-*`. Patched with [servo/html5ever#800](https://github.com/servo/html5ever/pull/800) by Josef Guenther ("Fix quadratic parsing of deeply nested markup", commit `67c7e7c423c5d1acab1872c630cae61e2bc45211`), which is not merged yet. The -PR is against html5ever 0.40.1; its `src/` changes apply unchanged to -0.39.0, apart from line offsets. Like the rest of html5ever, they are -available under MIT or Apache-2.0. +PR is against html5ever 0.40.1, and its `src/` changes are applied as they +are. Like the rest of html5ever, they are available under MIT or +Apache-2.0. Without it, tree building is quadratic in nesting depth (see [servo/html5ever#788](https://github.com/servo/html5ever/issues/788)): most @@ -28,16 +28,18 @@ in foreign content quadratic: at 32,000 levels, `html2text()` took 3.4 s on nested `` followed by unmatched ``, and 4.5 s on SVG `` followed by unmatched ``. Both now take 0.02 s or less. -Checked against stock 0.39.0 before vendoring: +Checked against stock 0.40.1 before vendoring: - html5ever's own tests pass, including the PR's new test, and so do all - 3,515 html5lib tree-construction tests, in release and debug builds. -- The tree, quirks mode and exact parse errors are identical to stock 0.39.0 - on every html5lib input (documents and fragments), 300,000 random tag-soup - records (30% of them fragments, in 23 contexts) and 28 real pages, - including the single-page HTML standard. -- pyhtml2text's output is byte-for-byte unchanged on the benchmark corpus - (`bench/bench.py --check`). + 3,515 html5lib tree-construction tests and the tokenizer tests, in + release and debug builds. +- The tree, quirks mode and exact parse errors are identical to stock 0.40.1 + on every html5lib tree-construction input (with scripting on and off), + 73,000 random tag-soup documents (a third of them fragments, in 25 + contexts) and 39 real pages, plus the single-page HTML standard. +- pyhtml2text's output, text and Markdown, is byte-for-byte the same as + with the previous copy (0.39.0 with the same PR): on those pages with + every option set, and on 13,000 of the random documents with six. The extra bookkeeping costs about 0.4 µs per call, and 0–2% on real pages. diff --git a/vendor/html5ever/Cargo.toml b/vendor/html5ever/Cargo.toml index 9dccdc1..34af598 100644 --- a/vendor/html5ever/Cargo.toml +++ b/vendor/html5ever/Cargo.toml @@ -11,9 +11,9 @@ [package] edition = "2021" -rust-version = "1.71.0" +rust-version = "1.85" name = "html5ever" -version = "0.39.0" +version = "0.40.1" authors = ["The html5ever Project Developers"] build = false autolib = false @@ -45,17 +45,12 @@ trace_tokenizer = [] name = "html5ever" path = "src/lib.rs" - - - - - - - [dependencies.log] version = "0.4" [dependencies.markup5ever] -version = "0.39" +version = "0.40" +[dependencies.memchr] +version = "2.8.0" diff --git a/vendor/html5ever/src/encoding.rs b/vendor/html5ever/src/encoding.rs index 19f5845..3e25781 100644 --- a/vendor/html5ever/src/encoding.rs +++ b/vendor/html5ever/src/encoding.rs @@ -36,7 +36,7 @@ pub(crate) fn extract_a_character_encoding_from_a_meta_element( // Step 4. If the next character is not a U+003D EQUALS SIGN (=), then move position to point just before // that next character, and jump back to the step labeled loop. - if input.as_bytes()[position] == b'=' { + if *input.as_bytes().get(position)? == b'=' { break; } } @@ -170,4 +170,54 @@ mod tests { Some(StrTendril::from_slice("utf8")) ); } + + #[test] + fn meta_element_with_truncated_charset() { + assert_eq!( + extract_a_character_encoding_from_a_meta_element(StrTendril::from_slice( + "text/html; charset" + )), + None + ); + assert_eq!( + extract_a_character_encoding_from_a_meta_element(StrTendril::from_slice( + "text/html; charset" + )), + None + ); + assert_eq!( + extract_a_character_encoding_from_a_meta_element(StrTendril::from_slice( + "text/html; charset \t" + )), + None + ); + assert_eq!( + extract_a_character_encoding_from_a_meta_element(StrTendril::from_slice( + "text/html; charset=" + )), + None + ); + assert_eq!( + extract_a_character_encoding_from_a_meta_element(StrTendril::from_slice( + "text/html; charset='" + )), + None + ); + assert_eq!( + extract_a_character_encoding_from_a_meta_element(StrTendril::from_slice( + "text/html; charset=\"" + )), + None + ); + } + + #[test] + fn meta_element_with_invalid_charset_followed_by_valid_charset() { + assert_eq!( + extract_a_character_encoding_from_a_meta_element(StrTendril::from_slice( + "text/html; charset charset=utf8" + )), + Some(StrTendril::from_slice("utf8")) + ); + } } diff --git a/vendor/html5ever/src/serialize/mod.rs b/vendor/html5ever/src/serialize/mod.rs index 9aae3ab..7cab15b 100644 --- a/vendor/html5ever/src/serialize/mod.rs +++ b/vendor/html5ever/src/serialize/mod.rs @@ -10,6 +10,7 @@ use log::warn; pub use markup5ever::serialize::{AttrRef, Serialize, Serializer, TraversalScope}; use markup5ever::{local_name, ns}; +use memchr::{memchr2, memchr3}; use std::io::{self, Write}; use crate::{LocalName, QualName}; @@ -102,16 +103,49 @@ impl HtmlSerializer { } fn write_escaped(&mut self, text: &str, attr_mode: bool) -> io::Result<()> { - for c in text.chars() { - match c { - '&' => self.writer.write_all(b"&"), - '\u{00A0}' => self.writer.write_all(b" "), - '"' if attr_mode => self.writer.write_all(b"""), - '<' => self.writer.write_all(b"<"), - '>' => self.writer.write_all(b">"), - c => self.writer.write_fmt(format_args!("{c}")), - }?; + // When in attribute mode quotes are escaped, but otherwise not. In order to reduce + // branching below, when not in attribute mode, just look for the another of the + // escaped characters. + let maybe_quote = if attr_mode { b'"' } else { b'<' }; + let find_next_escaped_character = |slice: &[u8]| { + // Use highly-optimized memchr to find the next character that needs to be escaped. + // Doing this twice is *much* faster than walking the string by characters. + let result = memchr3(maybe_quote, b'<', b'>', slice).unwrap_or(slice.len()); + memchr2(b'&', 0xC2, &slice[..result]).unwrap_or(result) + }; + + let bytes = text.as_bytes(); + let mut search_start = 0; + while search_start < text.len() { + let next_special = find_next_escaped_character(&bytes[search_start..]) + search_start; + + // Write all text before the search result unconditionally. + self.writer.write_all(&bytes[search_start..next_special])?; + + // If we reached the end of the text we can stop processing. + if next_special == bytes.len() { + break; + } + + search_start = next_special + 1; + let replacement = match bytes[next_special] { + b'&' => "&", + b'"' => """, + b'<' => "<", + b'>' => ">", + 0xC2 if bytes.get(next_special + 1) == Some(&0xA0) => { + search_start += 1; + " " + }, + _ => { + // 0xC2 not followed by 0xA0 (not NBSP), so keep looking. + self.writer.write_all(&bytes[next_special..search_start])?; + continue; + }, + }; + self.writer.write_all(replacement.as_bytes())?; } + Ok(()) } } diff --git a/vendor/html5ever/src/tokenizer/mod.rs b/vendor/html5ever/src/tokenizer/mod.rs index 7dd690e..b388fe2 100644 --- a/vendor/html5ever/src/tokenizer/mod.rs +++ b/vendor/html5ever/src/tokenizer/mod.rs @@ -14,10 +14,11 @@ pub use self::interface::{CommentToken, DoctypeToken, TagToken, Token}; pub use self::interface::{Doctype, EndTag, StartTag, Tag, TagKind}; pub use self::interface::{TokenSink, TokenSinkResult}; -use self::states::{DoctypeIdKind, Public, System}; -use self::states::{DoubleEscaped, Escaped}; -use self::states::{DoubleQuoted, SingleQuoted, Unquoted}; -use self::states::{Rawtext, Rcdata, ScriptData, ScriptDataEscaped}; +use self::states::AttrValueKind::*; +use self::states::DoctypeIdKind::{self, *}; +use self::states::RawKind::*; +use self::states::ScriptEscapeKind::*; +use self::states::State::{self, *}; use self::char_ref::{CharRef, CharRefTokenizer}; @@ -27,6 +28,7 @@ use log::{debug, trace}; use markup5ever::{ns, small_char_set, TokenizerResult}; use std::borrow::Cow::{self, Borrowed}; use std::cell::{Cell, RefCell, RefMut}; +use std::cmp::Reverse; use std::collections::BTreeMap; use std::mem; @@ -539,25 +541,22 @@ impl Tokenizer { if self.current_attr_name.borrow().is_empty() { return; } - + let name = LocalName::from(&**self.current_attr_name.borrow()); + self.current_attr_name.borrow_mut().clear(); // Check for a duplicate attribute. // FIXME: the spec says we should error as soon as the name is finished. let dup = { - let name = &*self.current_attr_name.borrow(); self.current_tag_attrs .borrow() .iter() - .any(|a| *a.name.local == **name) + .any(|a| a.name.local == name) }; if dup { self.emit_error(Borrowed("Duplicate attribute")); self.current_tag_had_duplicate_attributes.set(true); - self.current_attr_name.borrow_mut().clear(); self.current_attr_value.borrow_mut().clear(); } else { - let name = LocalName::from(&**self.current_attr_name.borrow()); - self.current_attr_name.borrow_mut().clear(); self.current_tag_attrs.borrow_mut().push(Attribute { // The tree builder will adjust the namespace if necessary. // This only happens in foreign elements. @@ -676,15 +675,9 @@ macro_rules! go ( // These can only come at the end. - ( $me:ident : to $s:ident ) => ({ $me.state.set(states::$s); return ProcessResult::Continue; }); - ( $me:ident : to $s:ident $k1:expr ) => ({ $me.state.set(states::$s($k1)); return ProcessResult::Continue; }); - ( $me:ident : to $s:ident $k1:ident $k2:expr ) => ({ $me.state.set(states::$s($k1($k2))); return ProcessResult::Continue; }); - - ( $me:ident : reconsume $s:ident ) => ({ $me.reconsume.set(true); go!($me: to $s); }); - ( $me:ident : reconsume $s:ident $k1:expr ) => ({ $me.reconsume.set(true); go!($me: to $s $k1); }); - ( $me:ident : reconsume $s:ident $k1:ident $k2:expr ) => ({ $me.reconsume.set(true); go!($me: to $s $k1 $k2); }); - - ( $me:ident : consume_char_ref ) => ({ $me.start_consuming_character_reference(); return ProcessResult::Continue; }); + ( $me:ident : to $s:expr ) => ({ $me.state.set($s); return ProcessResult::Continue; }); + ( $me:ident : reconsume $s:expr ) => ({ $me.reconsume.set(true); go!($me: to $s); }); + ( $me:ident : consume_char_ref ) => ({ $me.start_consuming_character_reference(); return ProcessResult::Continue;}); // We have a default next state after emitting a tag, but the sink can override. ( $me:ident : emit_tag $s:ident ) => ({ @@ -711,10 +704,6 @@ macro_rules! peek ( ($me:expr, $input:expr) => ( unwrap_or_return!($me.peek($input), ProcessResult::Suspend) )); -macro_rules! pop_except_from ( ($me:expr, $input:expr, $set:expr) => ( - unwrap_or_return!($me.pop_except_from($input, $set), ProcessResult::Suspend) -)); - macro_rules! eat ( ($me:expr, $input:expr, $pat:expr) => ( unwrap_or_return!($me.eat($input, $pat, u8::eq_ignore_ascii_case), ProcessResult::Suspend) )); @@ -735,8 +724,9 @@ impl Tokenizer { trace!("processing in state {:?}", self.state); match self.state.get() { - //§ data-state + // https://html.spec.whatwg.org/#data-state states::Data => loop { + // Step 1. Consume the next input character: let set = small_char_set!('\r' '\0' '&' '<' '\n'); #[cfg(any(target_arch = "x86", target_arch = "x86_64", target_arch = "aarch64"))] @@ -787,705 +777,1518 @@ impl Tokenizer { return ProcessResult::Suspend; }; match set_result { + // ↪ U+0026 AMPERSAND (&) + FromSet('&') => { + // Set the return state to the data state. Switch to the character reference state. + go!(self: consume_char_ref) + }, + // ↪ U+003C LESS-THAN SIGN (<) + FromSet('<') => { + // Switch to the tag open state. + go!(self: to State::TagOpen) + }, + // ↪ U+0000 NULL FromSet('\0') => { + // This is an unexpected-null-character parse error. + // Emit the current input character as a character token. self.bad_char_error(); self.emit_char('\0'); }, - FromSet('&') => go!(self: consume_char_ref), - FromSet('<') => go!(self: to TagOpen), - FromSet(c) => { - self.emit_char(c); - }, - NotFromSet(b) => self.emit_chars(b), + // ↪ Anything else + // Emit the current input character as a character token. + FromSet(character) => self.emit_char(character), + NotFromSet(characters) => self.emit_chars(characters), } }, - //§ rcdata-state + // https://html.spec.whatwg.org/#rcdata-state states::RawData(Rcdata) => loop { - match pop_except_from!(self, input, small_char_set!('\r' '\0' '&' '<' '\n')) { + // Consume the next input character: + let Some(set_result) = + self.pop_except_from(input, small_char_set!('\r' '\0' '&' '<' '\n')) + else { + return ProcessResult::Suspend; + }; + + match set_result { + // ↪ U+0026 AMPERSAND (&) + FromSet('&') => { + go!(self: consume_char_ref) + }, + // ↪ U+003C LESS-THAN SIGN (<) + FromSet('<') => { + // Switch to the RCDATA less-than sign state. + go!(self: to State::RawLessThanSign(Rcdata)) + }, + // ↪ U+0000 NULL FromSet('\0') => { self.bad_char_error(); self.emit_char('\u{fffd}'); }, - FromSet('&') => go!(self: consume_char_ref), - FromSet('<') => go!(self: to RawLessThanSign Rcdata), - FromSet(c) => self.emit_char(c), - NotFromSet(b) => self.emit_chars(b), + // ↪ Anything else + // Emit the current input character as a character token. + FromSet(character) => self.emit_char(character), + NotFromSet(characters) => self.emit_chars(characters), } }, - //§ rawtext-state + // https://html.spec.whatwg.org/#rawtext-state states::RawData(Rawtext) => loop { - match pop_except_from!(self, input, small_char_set!('\r' '\0' '<' '\n')) { + // Consume the next input character: + let Some(set_result) = + self.pop_except_from(input, small_char_set!('\r' '\0' '<' '\n')) + else { + return ProcessResult::Suspend; + }; + + match set_result { + // ↪ U+003C LESS-THAN SIGN (<) + FromSet('<') => { + // Switch to the RAWTEXT less-than sign state. + go!(self: to State::RawLessThanSign(Rawtext)); + }, + // ↪ U+0000 NULL FromSet('\0') => { + // This is an unexpected-null-character parse error. + // Emit a U+FFFD REPLACEMENT CHARACTER character token. self.bad_char_error(); self.emit_char('\u{fffd}'); }, - FromSet('<') => go!(self: to RawLessThanSign Rawtext), - FromSet(c) => self.emit_char(c), - NotFromSet(b) => self.emit_chars(b), + // ↪ Anything else + // Emit the current input character as a character token. + FromSet(character) => self.emit_char(character), + NotFromSet(characters) => self.emit_chars(characters), } }, - //§ script-data-state + // https://html.spec.whatwg.org/#script-data-state states::RawData(ScriptData) => loop { - match pop_except_from!(self, input, small_char_set!('\r' '\0' '<' '\n')) { - FromSet('\0') => { - self.bad_char_error(); - self.emit_char('\u{fffd}'); - }, - FromSet('<') => go!(self: to RawLessThanSign ScriptData), - FromSet(c) => self.emit_char(c), - NotFromSet(b) => self.emit_chars(b), - } - }, + // Consume the next input character: + let Some(set_result) = + self.pop_except_from(input, small_char_set!('\r' '\0' '<' '\n')) + else { + return ProcessResult::Suspend; + }; - //§ script-data-escaped-state - states::RawData(ScriptDataEscaped(Escaped)) => loop { - match pop_except_from!(self, input, small_char_set!('\r' '\0' '-' '<' '\n')) { - FromSet('\0') => { - self.bad_char_error(); - self.emit_char('\u{fffd}'); - }, - FromSet('-') => { - self.emit_char('-'); - go!(self: to ScriptDataEscapedDash Escaped); + match set_result { + // ↪ U+003C LESS-THAN SIGN (<) + FromSet('<') => { + // Switch to the script data less-than sign state. + go!(self: to State::RawLessThanSign(ScriptData)); }, - FromSet('<') => go!(self: to RawLessThanSign ScriptDataEscaped Escaped), - FromSet(c) => self.emit_char(c), - NotFromSet(b) => self.emit_chars(b), - } - }, - - //§ script-data-double-escaped-state - states::RawData(ScriptDataEscaped(DoubleEscaped)) => loop { - match pop_except_from!(self, input, small_char_set!('\r' '\0' '-' '<' '\n')) { + // ↪ U+0000 NULL FromSet('\0') => { + // This is an unexpected-null-character parse error. + // Emit a U+FFFD REPLACEMENT CHARACTER character token. self.bad_char_error(); self.emit_char('\u{fffd}'); }, - FromSet('-') => { - self.emit_char('-'); - go!(self: to ScriptDataEscapedDash DoubleEscaped); - }, - FromSet('<') => { - self.emit_char('<'); - go!(self: to RawLessThanSign ScriptDataEscaped DoubleEscaped) - }, - FromSet(c) => self.emit_char(c), - NotFromSet(b) => self.emit_chars(b), + // ↪ Anything else + // Emit the current input character as a character token. + FromSet(character) => self.emit_char(character), + NotFromSet(characters) => self.emit_chars(characters), } }, - //§ plaintext-state + // https://html.spec.whatwg.org/#plaintext-state states::Plaintext => loop { - match pop_except_from!(self, input, small_char_set!('\r' '\0' '\n')) { + // Consume the next input character: + let Some(set_result) = self.pop_except_from(input, small_char_set!('\r' '\0' '\n')) + else { + return ProcessResult::Suspend; + }; + + match set_result { + // ↪ U+0000 NULL FromSet('\0') => { + // This is an unexpected-null-character parse error. + // Emit a U+FFFD REPLACEMENT CHARACTER character token. self.bad_char_error(); self.emit_char('\u{fffd}'); }, - FromSet(c) => self.emit_char(c), - NotFromSet(b) => self.emit_chars(b), + // ↪ Anything else + // Emit the current input character as a character token. + FromSet(character) => self.emit_char(character), + NotFromSet(characters) => self.emit_chars(characters), } }, - //§ tag-open-state + // https://html.spec.whatwg.org/#tag-open-state states::TagOpen => loop { + // Consume the next input character: match get_char!(self, input) { - '!' => go!(self: to MarkupDeclarationOpen), - '/' => go!(self: to EndTagOpen), + // ↪ U+0021 EXCLAMATION MARK (!) + '!' => { + // Switch to the markup declaration open state. + go!(self: to State::MarkupDeclarationOpen) + }, + // ↪ U+002F SOLIDUS (/) + '/' => { + // Switch to the end tag open state. + go!(self: to State::EndTagOpen) + }, + // ↪ ASCII alpha + character if character.is_ascii_alphabetic() => { + // Create a new start tag token, set its tag name to the empty string. + // Reconsume in the tag name state. + // NOTE: We don't reconsume the character but instead immediately append it in lowercase + // to the new tag (as that is what the "tag name" state would do). + let character = character.to_ascii_lowercase(); + go!(self: create_tag StartTag character; to State::TagName) + }, + // ↪ U+003F QUESTION MARK (?) '?' => { + // Set the temporary buffer to the empty string. + // Switch to the processing instruction open state. self.bad_char_error(); go!(self: clear_comment; reconsume BogusComment) }, - c => match lower_ascii_letter(c) { - Some(cl) => go!(self: create_tag StartTag cl; to TagName), - None => { - self.bad_char_error(); - self.emit_char('<'); - go!(self: reconsume Data) - }, + // ↪ Anything else + _ => { + // This is an invalid-first-character-of-tag-name parse error. + // Emit a U+003C LESS-THAN SIGN character token. + // Reconsume in the data state. + self.bad_char_error(); + self.emit_char('<'); + go!(self: reconsume Data) }, } }, - //§ end-tag-open-state + // https://html.spec.whatwg.org/#end-tag-open-state states::EndTagOpen => loop { + // Consume the next input character: match get_char!(self, input) { + // ↪ ASCII alpha + character if character.is_ascii_alphabetic() => { + // Create a new end tag token, set its tag name to the empty string. + // Reconsume in the tag name state. + // NOTE: We don't reconsume the character but instead immediately append in lowercase + // to the new tag (as that is what the "tag name" state would do). + let character = character.to_ascii_lowercase(); + go!(self: create_tag EndTag character; to State::TagName) + }, + // ↪ U+003E GREATER-THAN SIGN (>) '>' => { + // This is a missing-end-tag-name parse error. + // Switch to the data state. self.bad_char_error(); - go!(self: to Data) + go!(self: to State::Data) }, - c => match lower_ascii_letter(c) { - Some(cl) => go!(self: create_tag EndTag cl; to TagName), - None => { - self.bad_char_error(); - go!(self: clear_comment; reconsume BogusComment) - }, + // ↪ Anything else + _ => { + // This is an invalid-first-character-of-tag-name parse error. + // Create a comment token whose data is the empty string. + // Reconsume in the bogus comment state. + self.bad_char_error(); + go!(self: clear_comment; reconsume BogusComment) }, } }, - //§ tag-name-state + // https://html.spec.whatwg.org/#tag-name-state states::TagName => loop { + // Consume the next input character: match get_char!(self, input) { - '\t' | '\n' | '\x0C' | ' ' => go!(self: to BeforeAttributeName), - '/' => go!(self: to SelfClosingStartTag), - '>' => go!(self: emit_tag Data), + // ↪ U+0009 CHARACTER TABULATION (tab) + // ↪ U+000A LINE FEED (LF) + // ↪ U+000C FORM FEED (FF) + // ↪ U+0020 SPACE + '\t' | '\n' | '\x0C' | ' ' => { + // Switch to the before attribute name state. + go!(self: to State::BeforeAttributeName) + }, + // ↪ U+002F SOLIDUS (/) + '/' => { + // Switch to the self-closing start tag state. + go!(self: to State::SelfClosingStartTag) + }, + // ↪ U+003E GREATER-THAN SIGN (>) + '>' => { + // Switch to the data state. + // Emit the current tag token. + go!(self: emit_tag Data) + }, + // ↪ ASCII upper alpha + character if character.is_ascii_uppercase() => { + // Append the lowercase version of the current input character (add 0x0020 to the + // character's code point) to the current tag token's tag name. + go!(self: push_tag (character.to_ascii_lowercase())) + }, + // ↪ U+0000 NULL '\0' => { self.bad_char_error(); go!(self: push_tag '\u{fffd}') }, - c => go!(self: push_tag (c.to_ascii_lowercase())), + // ↪ Anything else + character => { + // Append the current input character to the current tag token's tag name. + go!(self: push_tag (character)) + }, } }, - //§ script-data-escaped-less-than-sign-state + // https://html.spec.whatwg.org/#script-data-escaped-less-than-sign-state states::RawLessThanSign(ScriptDataEscaped(Escaped)) => loop { + // Consume the next input character: match get_char!(self, input) { - '/' => go!(self: clear_temp; to RawEndTagOpen ScriptDataEscaped Escaped), - c => match lower_ascii_letter(c) { - Some(cl) => { - go!(self: clear_temp; push_temp cl); + // ↪ U+002F SOLIDUS (/) + '/' => { + go!(self: clear_temp; to State::RawEndTagOpen(ScriptDataEscaped(Escaped))) + }, + character => match lower_ascii_letter(character) { + // ↪ ASCII alpha + Some(character_lowercase) => { + // Set the temporary buffer to the empty string. + // Emit a U+003C LESS-THAN SIGN character token. + // Reconsume in the script data double escape start state. + // NOTE: We don't reconsume, and instead emit the lowercased character immediately, + // as that is what the "script data double escape start" state would do. + go!(self: clear_temp; push_temp character_lowercase); self.emit_char('<'); - self.emit_char(c); - go!(self: to ScriptDataEscapeStart DoubleEscaped); + self.emit_char(character); + go!(self: to State::ScriptDataEscapeStart(DoubleEscaped)); }, + // ↪ Anything else None => { self.emit_char('<'); - go!(self: reconsume RawData ScriptDataEscaped Escaped); + go!(self: reconsume RawData(ScriptDataEscaped(Escaped))); }, }, } }, - //§ script-data-double-escaped-less-than-sign-state + // https://html.spec.whatwg.org/#script-data-double-escaped-less-than-sign-state states::RawLessThanSign(ScriptDataEscaped(DoubleEscaped)) => loop { + // Consume the next input character: match get_char!(self, input) { + // ↪ U+002F SOLIDUS (/) '/' => { + // Set the temporary buffer to the empty string. + // Switch to the script data double escape end state. + // Emit a U+002F SOLIDUS character token. go!(self: clear_temp); self.emit_char('/'); - go!(self: to ScriptDataDoubleEscapeEnd); + go!(self: to State::ScriptDataDoubleEscapeEnd); + }, + // ↪ Anything else + _ => { + // Reconsume in the script data double escaped state. + go!(self: reconsume RawData(ScriptDataEscaped(DoubleEscaped))) }, - _ => go!(self: reconsume RawData ScriptDataEscaped DoubleEscaped), } }, - //§ rcdata-less-than-sign-state rawtext-less-than-sign-state script-data-less-than-sign-state - // otherwise + // https://html.spec.whatwg.org/#rcdata-less-than-sign-state + // https://html.spec.whatwg.org/#script-data-less-than-sign-state + // https://html.spec.whatwg.org/#rawtext-less-than-sign-state states::RawLessThanSign(kind) => loop { + // Consume the next input character: match get_char!(self, input) { - '/' => go!(self: clear_temp; to RawEndTagOpen kind), + // ↪ U+002F SOLIDUS (/) + '/' => { + // Set the temporary buffer to the empty string. + // Switch to the RCDATA end tag open state. + go!(self: clear_temp; to State::RawEndTagOpen(kind)) + }, + // ↪ U+0021 EXCLAMATION MARK (!) '!' if kind == ScriptData => { + // Switch to the script data escape start state. + // Emit a U+003C LESS-THAN SIGN character token and a U+0021 EXCLAMATION MARK character token. self.emit_char('<'); self.emit_char('!'); - go!(self: to ScriptDataEscapeStart Escaped); + go!(self: to State::ScriptDataEscapeStart(Escaped)); }, + // ↪ Anything else _ => { + // Emit a U+003C LESS-THAN SIGN character token. + // Reconsume in the RCDATA/script data/RAWTEXT state. self.emit_char('<'); - go!(self: reconsume RawData kind); + go!(self: reconsume RawData(kind)); }, } }, - //§ rcdata-end-tag-open-state rawtext-end-tag-open-state script-data-end-tag-open-state script-data-escaped-end-tag-open-state + // https://html.spec.whatwg.org/#rcdata-end-tag-open-state + // https://html.spec.whatwg.org/#rawtext-end-tag-open-state + // https://html.spec.whatwg.org/#script-data-end-tag-open-state + // https://html.spec.whatwg.org/#script-data-escaped-end-tag-open-state states::RawEndTagOpen(kind) => loop { - let c = get_char!(self, input); - match lower_ascii_letter(c) { - Some(cl) => go!(self: create_tag EndTag cl; push_temp c; to RawEndTagName kind), - None => { - self.emit_char('<'); - self.emit_char('/'); - go!(self: reconsume RawData kind); - }, - } - }, - - //§ rcdata-end-tag-name-state rawtext-end-tag-name-state script-data-end-tag-name-state script-data-escaped-end-tag-name-state + // Consume the next input character: + let character = get_char!(self, input); + + // ↪ ASCII alpha + if character.is_ascii_alphabetic() { + // Create a new end tag token, set its tag name to the empty string. + // Reconsume in the RCDATA end tag name state. + // NOTE: We don't reconsume the character but instead immediately append its lowercase version + // to the end tag (as that is what the new state would do). + let character_lowercase = character.to_ascii_lowercase(); + go!(self: create_tag EndTag character_lowercase; push_temp character; to State::RawEndTagName(kind)) + } + // ↪ Anything else + else { + // Emit a U+003C LESS-THAN SIGN character token and a U+002F SOLIDUS character token. + // Reconsume in the RCDATA/RAWTEXT/script data/script data escaped state. + self.emit_char('<'); + self.emit_char('/'); + go!(self: reconsume RawData(kind)); + } + }, + + // https://html.spec.whatwg.org/#rcdata-end-tag-name-state + // https://html.spec.whatwg.org/#rawtext-end-tag-name-state + // https://html.spec.whatwg.org/#script-data-end-tag-name-state + // https://html.spec.whatwg.org/#script-data-escaped-end-tag-name-state states::RawEndTagName(kind) => loop { - let c = get_char!(self, input); + let character = get_char!(self, input); + + // NOTE: The first three match arms in the specification are treated as "anything else" if the current + // end tag token is NOT an appropriate end tag, so we move them into their own match block. if self.have_appropriate_end_tag() { - match c { - '\t' | '\n' | '\x0C' | ' ' => go!(self: clear_temp; to BeforeAttributeName), - '/' => go!(self: clear_temp; to SelfClosingStartTag), - '>' => go!(self: clear_temp; emit_tag Data), - _ => (), + match character { + // ↪ U+0009 CHARACTER TABULATION (tab) + // ↪ U+000A LINE FEED (LF) + // ↪ U+000C FORM FEED (FF) + // ↪ U+0020 SPACE + '\t' | '\n' | '\x0C' | ' ' => { + // Switch to the before attribute name state + go!(self: clear_temp; to State::BeforeAttributeName) + }, + // ↪ U+002F SOLIDUS (/) + '/' => { + // Switch to the self-closing start tag state + go!(self: clear_temp; to State::SelfClosingStartTag) + }, + // ↪ U+003E GREATER-THAN SIGN (>) + '>' => { + // Switch to the data state and emit the current tag token + go!(self: clear_temp; emit_tag Data) + }, + _ => {}, } } - match lower_ascii_letter(c) { - Some(cl) => go!(self: push_tag cl; push_temp c), + match lower_ascii_letter(character) { + // ↪ ASCII upper alpha + // NOTE: This is the same as for lower alpha, but the character is lowercased first. + // This is handled by lower_ascii_letter. + // ↪ ASCII lower alpha + Some(character_lowercase) => { + // Append the current input character to the current tag token's tag name. + // Append the current input character to the temporary buffer. + go!(self: push_tag character_lowercase; push_temp character) + }, + // ↪ Anything else None => { + // Emit a U+003C LESS-THAN SIGN character token, a U+002F SOLIDUS character token, + // and a character token for each of the characters in the temporary buffer + // (in the order they were added to the buffer). + // Reconsume in the RCDATA/RAWTEXT/script data/script data escaped state. go!(self: discard_tag); self.emit_char('<'); self.emit_char('/'); self.emit_temp_buf(); - go!(self: reconsume RawData kind); + go!(self: reconsume RawData(kind)); }, } }, - //§ script-data-double-escape-start-state + // https://html.spec.whatwg.org/#script-data-double-escape-start-state states::ScriptDataEscapeStart(DoubleEscaped) => loop { - let c = get_char!(self, input); - match c { - '\t' | '\n' | '\x0C' | ' ' | '/' | '>' => { - let esc = if &**self.temp_buf.borrow() == "script" { + // Consume the next input character: + match get_char!(self, input) { + // ↪ U+0009 CHARACTER TABULATION (tab) + // ↪ U+000A LINE FEED (LF) + // ↪ U+000C FORM FEED (FF) + // ↪ U+0020 SPACE + // ↪ U+002F SOLIDUS (/) + // ↪ U+003E GREATER-THAN SIGN (>) + character @ ('\t' | '\n' | '\x0C' | ' ' | '/' | '>') => { + // If the temporary buffer is "script", then switch to the script data double escaped state. + // Otherwise, switch to the script data escaped state. + // Emit the current input character as a character token. + let escaped_kind = if &**self.temp_buf.borrow() == "script" { DoubleEscaped } else { Escaped }; - self.emit_char(c); - go!(self: to RawData ScriptDataEscaped esc); - }, - _ => match lower_ascii_letter(c) { - Some(cl) => { - go!(self: push_temp cl); - self.emit_char(c); + self.emit_char(character); + go!(self: to State::RawData(ScriptDataEscaped(escaped_kind))); + }, + // ↪ ASCII upper alpha + // NOTE: This is the same as the "ASCII lower alpha" branch, except we lowercase the character first + // ↪ ASCII lower alpha + character => match lower_ascii_letter(character) { + Some(character_lowercase) => { + // Append the current input character to the temporary buffer. + // Emit the current input character as a character token. + go!(self: push_temp character_lowercase); + self.emit_char(character); + }, + // ↪ Anything else + None => { + // Reconsume in the script data escaped state. + go!(self: reconsume RawData(ScriptDataEscaped(Escaped))) }, - None => go!(self: reconsume RawData ScriptDataEscaped Escaped), }, } }, - //§ script-data-escape-start-state + // https://html.spec.whatwg.org/#script-data-double-escaped-state + states::RawData(ScriptDataEscaped(DoubleEscaped)) => loop { + // Consume the next input character: + let Some(set_result) = + self.pop_except_from(input, small_char_set!('\r' '\0' '-' '<' '\n')) + else { + return ProcessResult::Suspend; + }; + + match set_result { + // ↪ U+002D HYPHEN-MINUS (-) + FromSet('-') => { + // Switch to the script data double escaped dash state. + // Emit a U+002D HYPHEN-MINUS character token. + self.emit_char('-'); + go!(self: to State::ScriptDataEscapedDash(DoubleEscaped)); + }, + // ↪ U+003C LESS-THAN SIGN (<) + FromSet('<') => { + // Switch to the script data double escaped less-than sign state. + // Emit a U+003C LESS-THAN SIGN character token. + self.emit_char('<'); + go!(self: to State::RawLessThanSign(ScriptDataEscaped(DoubleEscaped))) + }, + // ↪ U+0000 NULL + FromSet('\0') => { + // This is an unexpected-null-character parse error. + // Emit a U+FFFD REPLACEMENT CHARACTER character token. + self.bad_char_error(); + self.emit_char('\u{fffd}'); + }, + // ↪ Anything else + // Emit the current input character as a character token. + FromSet(character) => self.emit_char(character), + NotFromSet(characters) => self.emit_chars(characters), + } + }, + + // https://html.spec.whatwg.org/#script-data-escape-start-state states::ScriptDataEscapeStart(Escaped) => loop { + // Consume the next input character: match get_char!(self, input) { + // ↪ U+002D HYPHEN-MINUS (-) '-' => { + // Switch to the script data escape start dash state. + // Emit a U+002D HYPHEN-MINUS character token. self.emit_char('-'); - go!(self: to ScriptDataEscapeStartDash); + go!(self: to State::ScriptDataEscapeStartDash); + }, + // ↪ Anything else + _ => { + // Reconsume in the script data state. + go!(self: reconsume RawData(ScriptData)) }, - _ => go!(self: reconsume RawData ScriptData), } }, - //§ script-data-escape-start-dash-state + // https://html.spec.whatwg.org/#script-data-escape-start-dash-state states::ScriptDataEscapeStartDash => loop { + // Consume the next input character: match get_char!(self, input) { + // ↪ U+002D HYPHEN-MINUS (-) '-' => { + // Switch to the script data escaped dash dash state. + // Emit a U+002D HYPHEN-MINUS character token. self.emit_char('-'); - go!(self: to ScriptDataEscapedDashDash Escaped); + go!(self: to State::ScriptDataEscapedDashDash(Escaped)); + }, + // ↪ Anything else + _ => { + // Reconsume in the script data state. + go!(self: reconsume RawData(ScriptData)) }, - _ => go!(self: reconsume RawData ScriptData), } }, - //§ script-data-escaped-dash-state script-data-double-escaped-dash-state + // https://html.spec.whatwg.org/#script-data-escaped-state + states::RawData(ScriptDataEscaped(Escaped)) => loop { + // Consume the next input character: + let Some(set_result) = + self.pop_except_from(input, small_char_set!('\r' '\0' '-' '<' '\n')) + else { + return ProcessResult::Suspend; + }; + + match set_result { + // ↪ U+002D HYPHEN-MINUS (-) + FromSet('-') => { + // Switch to the script data escaped dash state. + // Emit a U+002D HYPHEN-MINUS character token. + self.emit_char('-'); + go!(self: to State::ScriptDataEscapedDash(Escaped)); + }, + // ↪ U+003C LESS-THAN SIGN (<) + FromSet('<') => { + // Switch to the script data escaped less-than sign state. + go!(self: to State::RawLessThanSign(ScriptDataEscaped(Escaped))) + }, + // ↪ U+0000 NULL + FromSet('\0') => { + // This is an unexpected-null-character parse error. + // Emit a U+FFFD REPLACEMENT CHARACTER character token. + self.bad_char_error(); + self.emit_char('\u{fffd}'); + }, + // ↪ Anything else + // Emit the current input character as a character token. + FromSet(character) => self.emit_char(character), + NotFromSet(characters) => self.emit_chars(characters), + } + }, + + // https://html.spec.whatwg.org/#script-data-escaped-dash-state + // https://html.spec.whatwg.org/#script-data-double-escaped-dash-state states::ScriptDataEscapedDash(kind) => loop { + // Consume the next input character: match get_char!(self, input) { + // ↪ U+002D HYPHEN-MINUS (-) '-' => { + // Switch to the script data escaped dash dash/script data double escaped dash dash state. + // Emit a U+002D HYPHEN-MINUS character token. self.emit_char('-'); - go!(self: to ScriptDataEscapedDashDash kind); + go!(self: to State::ScriptDataEscapedDashDash(kind)); }, + // ↪ U+003C LESS-THAN SIGN (<) '<' => { + // Switch to the script data escaped less-than sign/script data double escaped less-than sign state. if kind == DoubleEscaped { self.emit_char('<'); } - go!(self: to RawLessThanSign ScriptDataEscaped kind); + go!(self: to State::RawLessThanSign(ScriptDataEscaped(kind))); }, + // ↪ U+0000 NULL '\0' => { + // This is an unexpected-null-character parse error. + // Switch to the script data escaped/script data double escaped state. + // Emit a U+FFFD REPLACEMENT CHARACTER character token. self.bad_char_error(); self.emit_char('\u{fffd}'); - go!(self: to RawData ScriptDataEscaped kind) + go!(self: to State::RawData(ScriptDataEscaped(kind))); }, + // ↪ Anything else c => { + // Switch to the script data escaped/script data double escaped state. + // Emit the current input character as a character token. self.emit_char(c); - go!(self: to RawData ScriptDataEscaped kind); + go!(self: to State::RawData(ScriptDataEscaped(kind))); }, } }, - //§ script-data-escaped-dash-dash-state script-data-double-escaped-dash-dash-state + // https://html.spec.whatwg.org/#script-data-escaped-dash-dash-state + // https://html.spec.whatwg.org/#script-data-double-escaped-dash-dash-state states::ScriptDataEscapedDashDash(kind) => loop { + // Consume the next input character: match get_char!(self, input) { + // ↪ U+002D HYPHEN-MINUS (-) '-' => { + // Emit a U+002D HYPHEN-MINUS character token. self.emit_char('-'); }, + // ↪ U+003C LESS-THAN SIGN (<) '<' => { + // Switch to the script data escaped less-than sign/script data double escaped less-than sign state. if kind == DoubleEscaped { self.emit_char('<'); } - go!(self: to RawLessThanSign ScriptDataEscaped kind); + go!(self: to State::RawLessThanSign(ScriptDataEscaped(kind))); }, + // ↪ U+003E GREATER-THAN SIGN (>) '>' => { + // Switch to the script data state. + // Emit a U+003E GREATER-THAN SIGN character token. self.emit_char('>'); - go!(self: to RawData ScriptData); + go!(self: to State::RawData(ScriptData)); }, + // ↪ U+0000 NULL '\0' => { + // This is an unexpected-null-character parse error. + // Switch to the script data escaped/script data double escaped state. + // Emit a U+FFFD REPLACEMENT CHARACTER character token. self.bad_char_error(); self.emit_char('\u{fffd}'); - go!(self: to RawData ScriptDataEscaped kind) + go!(self: to State::RawData(ScriptDataEscaped(kind))) }, - c => { - self.emit_char(c); - go!(self: to RawData ScriptDataEscaped kind); + // ↪ Anything else + character => { + // Switch to the script data escaped/script data double escaped state. + // Emit the current input character as a character token. + self.emit_char(character); + go!(self: to State::RawData(ScriptDataEscaped(kind))); }, } }, - //§ script-data-double-escape-end-state + // https://html.spec.whatwg.org/#script-data-double-escape-end-state states::ScriptDataDoubleEscapeEnd => loop { - let c = get_char!(self, input); - match c { - '\t' | '\n' | '\x0C' | ' ' | '/' | '>' => { - let esc = if &**self.temp_buf.borrow() == "script" { + // Consume the next input character: + match get_char!(self, input) { + // ↪ U+0009 CHARACTER TABULATION (tab) + // ↪ U+000A LINE FEED (LF) + // ↪ U+000C FORM FEED (FF) + // ↪ U+0020 SPACE + // ↪ U+002F SOLIDUS (/) + // ↪ U+003E GREATER-THAN SIGN (>) + character @ ('\t' | '\n' | '\x0C' | ' ' | '/' | '>') => { + // If the temporary buffer is "script", then switch to the script data escaped state. + // Otherwise, switch to the script data double escaped state. + // Emit the current input character as a character token. + let escaped_kind = if &**self.temp_buf.borrow() == "script" { Escaped } else { DoubleEscaped }; - self.emit_char(c); - go!(self: to RawData ScriptDataEscaped esc); - }, - _ => match lower_ascii_letter(c) { - Some(cl) => { - go!(self: push_temp cl); - self.emit_char(c); + self.emit_char(character); + go!(self: to State::RawData(ScriptDataEscaped(escaped_kind))); + }, + + character => match lower_ascii_letter(character) { + // ↪ ASCII upper alpha + // NOTE: This is the same as "ASCII lower alpha", except we lowercase the character first. + // That is handled by lower_ascii_letter. + // ↪ ASCII lower alpha + Some(character_lowercase) => { + // Append the current input character to the temporary buffer. + // Emit the current input character as a character token. + go!(self: push_temp character_lowercase); + self.emit_char(character); + }, + // ↪ Anything else + None => { + // Reconsume in the script data double escaped state. + go!(self: reconsume RawData(ScriptDataEscaped(DoubleEscaped))) }, - None => go!(self: reconsume RawData ScriptDataEscaped DoubleEscaped), }, } }, - //§ before-attribute-name-state + // https://html.spec.whatwg.org/#before-attribute-name-state states::BeforeAttributeName => loop { match get_char!(self, input) { - '\t' | '\n' | '\x0C' | ' ' => (), - '/' => go!(self: to SelfClosingStartTag), - '>' => go!(self: emit_tag Data), + // ↪ U+0009 CHARACTER TABULATION (tab) + // ↪ U+000A LINE FEED (LF) + // ↪ U+000C FORM FEED (FF) + // ↪ U+0020 SPACE + '\t' | '\n' | '\x0C' | ' ' => { + // Ignore the character. + }, + // U+002F SOLIDUS (/) + '/' => { + // Reconsume in the after attribute name state. + // NOTE: Instead we move to the self closing start tag, + // as that is what the "after attribute name" state would do. + go!(self: to State::SelfClosingStartTag) + }, + // U+003E GREATER-THAN SIGN (>) + '>' => { + // Reconsume in the after attribute name state. + // NOTE: Instead we emit the current tag and move to the data state, + // as that is what the "after attribute name" state would do. + go!(self: emit_tag Data) + }, + // NOTE: In the "anything else" case we should reconsume in the attribute name state, + // but instead of reconsuming we inline what that state *would* do here. '\0' => { self.bad_char_error(); - go!(self: create_attr '\u{fffd}'; to AttributeName) + go!(self: create_attr '\u{fffd}'; to State::AttributeName) }, - c => match lower_ascii_letter(c) { - Some(cl) => go!(self: create_attr cl; to AttributeName), + character => match lower_ascii_letter(character) { + Some(character) => { + go!(self: create_attr character; to State::AttributeName) + }, None => { - if matches!(c, '"' | '\'' | '<' | '=') { + if matches!(character, '"' | '\'' | '<' | '=') { self.bad_char_error(); } - go!(self: create_attr c; to AttributeName); + go!(self: create_attr character; to State::AttributeName); }, }, } }, - //§ attribute-name-state + // https://html.spec.whatwg.org/#attribute-name-state states::AttributeName => loop { + // Consume the next input character: match get_char!(self, input) { - '\t' | '\n' | '\x0C' | ' ' => go!(self: to AfterAttributeName), - '/' => go!(self: to SelfClosingStartTag), - '=' => go!(self: to BeforeAttributeValue), - '>' => go!(self: emit_tag Data), + // ↪ U+0009 CHARACTER TABULATION (tab) + // ↪ U+000A LINE FEED (LF) + // ↪ U+000C FORM FEED (FF) + // ↪ U+0020 SPACE + '\t' | '\n' | '\x0C' | ' ' => { + // Reconsume in the after attribute name state. + // NOTE: Instead we move to the after attribute name state and ignore + // the character, as that state would ignore it as well. + go!(self: to State::AfterAttributeName) + }, + // ↪ U+002F SOLIDUS (/) + '/' => { + // Reconsume in the after attribute name state. + // NOTE: Instead we move to the self closing start tag state, as that is + // what the after attribute name state would do when it encounters a '/'. + go!(self: to State::SelfClosingStartTag) + }, + // ↪ U+003E GREATER-THAN SIGN (>) + '>' => { + // Reconsume in the after attribute name state. + // NOTE: Instead we move to the data state, as that is + // what the after attribute name state would do when it encounters a '>'. + go!(self: emit_tag Data) + }, + // ↪ U+003D EQUALS SIGN (=) + '=' => { + // Switch to the before attribute value state. + go!(self: to State::BeforeAttributeValue) + }, + // ↪ U+0000 NULL '\0' => { + // This is an unexpected-null-character parse error. + // Append a U+FFFD REPLACEMENT CHARACTER character to the current attribute's name. self.bad_char_error(); go!(self: push_name '\u{fffd}') }, - c => match lower_ascii_letter(c) { - Some(cl) => go!(self: push_name cl), + character => match lower_ascii_letter(character) { + // ↪ ASCII upper alpha + Some(character_lowercase) => { + // Append the lowercase version of the current input character + // (add 0x0020 to the character's code point) to the current attribute's name. + go!(self: push_name character_lowercase) + }, None => { - if matches!(c, '"' | '\'' | '<') { + // ↪ U+0022 QUOTATION MARK (") + // ↪ U+0027 APOSTROPHE (') + // ↪ U+003C LESS-THAN SIGN (<) + if matches!(character, '"' | '\'' | '<') { + // This is an unexpected-character-in-attribute-name parse error. + // Treat it as per the "anything else" entry below. self.bad_char_error(); } - go!(self: push_name c); + // ↪ Anything else + // Append the current input character to the current attribute's name. + go!(self: push_name character); }, }, } }, - //§ after-attribute-name-state + // https://html.spec.whatwg.org/#after-attribute-name-state states::AfterAttributeName => loop { + // Consume the next input character: match get_char!(self, input) { - '\t' | '\n' | '\x0C' | ' ' => (), - '/' => go!(self: to SelfClosingStartTag), - '=' => go!(self: to BeforeAttributeValue), - '>' => go!(self: emit_tag Data), + // ↪ U+0009 CHARACTER TABULATION (tab) + // ↪ U+000A LINE FEED (LF) + // ↪ U+000C FORM FEED (FF) + // ↪ U+0020 SPACE + '\t' | '\n' | '\x0C' | ' ' => { + // Ignore the character. + }, + // ↪ U+002F SOLIDUS (/) + '/' => { + // Switch to the self-closing start tag state. + go!(self: to State::SelfClosingStartTag) + }, + // ↪ U+003D EQUALS SIGN (=) + '=' => { + // Switch to the before attribute value state. + go!(self: to State::BeforeAttributeValue) + }, + // ↪ U+003E GREATER-THAN SIGN (>) + '>' => { + // Switch to the data state. Emit the current tag token. + go!(self: emit_tag Data) + }, + // NOTE: The "anything else" match arm in the specification reconsumes + // the input in the attribute name state. Instead of reconsuming we inline + // what the attribute name state *would* do, and the move to it. '\0' => { self.bad_char_error(); - go!(self: create_attr '\u{fffd}'; to AttributeName) + go!(self: create_attr '\u{fffd}'; to State::AttributeName) }, - c => match lower_ascii_letter(c) { - Some(cl) => go!(self: create_attr cl; to AttributeName), + character => match lower_ascii_letter(character) { + Some(character_lowercase) => { + go!(self: create_attr character_lowercase; to State::AttributeName) + }, None => { - if matches!(c, '"' | '\'' | '<') { + if matches!(character, '"' | '\'' | '<') { self.bad_char_error(); } - go!(self: create_attr c; to AttributeName); + go!(self: create_attr character; to State::AttributeName); }, }, } }, - //§ before-attribute-value-state + // https://html.spec.whatwg.org/#before-attribute-value-state // Use peek so we can handle the first attr character along with the rest, // hopefully in the same zero-copy buffer. states::BeforeAttributeValue => loop { + // Consume the next input character: match peek!(self, input) { - '\t' | '\n' | '\r' | '\x0C' | ' ' => go!(self: discard_char input), - '"' => go!(self: discard_char input; to AttributeValue DoubleQuoted), - '\'' => go!(self: discard_char input; to AttributeValue SingleQuoted), + // ↪ U+0009 CHARACTER TABULATION (tab) + // ↪ U+000A LINE FEED (LF) + // ↪ U+000C FORM FEED (FF) + // ↪ U+0020 SPACE + '\t' | '\n' | '\r' | '\x0C' | ' ' => { + // Ignore the character. + go!(self: discard_char input) + }, + // ↪ U+0022 QUOTATION MARK (") + '"' => { + // Switch to the attribute value (double-quoted) state. + go!(self: discard_char input; to State::AttributeValue(DoubleQuoted)) + }, + // ↪ U+0027 APOSTROPHE (') + '\'' => { + // Switch to the attribute value (single-quoted) state. + go!(self: discard_char input; to State::AttributeValue(SingleQuoted)) + }, + // ↪ U+003E GREATER-THAN SIGN (>) '>' => { + // This is a missing-attribute-value parse error. + // Switch to the data state. + // Emit the current tag token. go!(self: discard_char input); self.bad_char_error(); go!(self: emit_tag Data) }, - _ => go!(self: to AttributeValue Unquoted), + // ↪ Anything else + _ => { + // Reconsume in the attribute value (unquoted) state. + go!(self: to State::AttributeValue(Unquoted)) + }, } }, - //§ attribute-value-(double-quoted)-state + // https://html.spec.whatwg.org/#attribute-value-(double-quoted)-state states::AttributeValue(DoubleQuoted) => loop { - match pop_except_from!(self, input, small_char_set!('\r' '"' '&' '\0' '\n')) { - FromSet('"') => go!(self: to AfterAttributeValueQuoted), - FromSet('&') => go!(self: consume_char_ref), + // Consume the next input character: + let Some(set_result) = + self.pop_except_from(input, small_char_set!('\r' '"' '&' '\0' '\n')) + else { + return ProcessResult::Suspend; + }; + + match set_result { + // ↪ U+0022 QUOTATION MARK (") + FromSet('"') => { + // Switch to the after attribute value (quoted) state. + go!(self: to State::AfterAttributeValueQuoted) + }, + // ↪ U+0026 AMPERSAND (&) + FromSet('&') => { + // Set the return state to the attribute value (double-quoted) state. + // Switch to the character reference state. + go!(self: consume_char_ref) + }, + // ↪ U+0000 NULL FromSet('\0') => { + // This is an unexpected-null-character parse error. + // Append a U+FFFD REPLACEMENT CHARACTER character to the current attribute's value. self.bad_char_error(); go!(self: push_value '\u{fffd}') }, - FromSet(c) => go!(self: push_value c), - NotFromSet(ref b) => go!(self: append_value b), + // ↪ Anything else + // Append the current input character to the current attribute's value. + FromSet(character) => go!(self: push_value character), + NotFromSet(ref characters) => go!(self: append_value characters), } }, - //§ attribute-value-(single-quoted)-state + // https://html.spec.whatwg.org/#attribute-value-(single-quoted)-state states::AttributeValue(SingleQuoted) => loop { - match pop_except_from!(self, input, small_char_set!('\r' '\'' '&' '\0' '\n')) { - FromSet('\'') => go!(self: to AfterAttributeValueQuoted), - FromSet('&') => go!(self: consume_char_ref), + // Consume the next input character: + let Some(set_result) = + self.pop_except_from(input, small_char_set!('\r' '\'' '&' '\0' '\n')) + else { + return ProcessResult::Suspend; + }; + + match set_result { + // ↪ U+0027 APOSTROPHE (') + FromSet('\'') => { + // Switch to the after attribute value (quoted) state. + go!(self: to State::AfterAttributeValueQuoted) + }, + // ↪ U+0026 AMPERSAND (&) + FromSet('&') => { + // Set the return state to the attribute value (single-quoted) state. + // Switch to the character reference state. + go!(self: consume_char_ref) + }, + // ↪ U+0000 NULL FromSet('\0') => { + // This is an unexpected-null-character parse error. + // Append a U+FFFD REPLACEMENT CHARACTER character to the current attribute's value. self.bad_char_error(); go!(self: push_value '\u{fffd}') }, - FromSet(c) => go!(self: push_value c), - NotFromSet(ref b) => go!(self: append_value b), + // ↪ Anything else + // Append the current input character to the current attribute's value. + FromSet(character) => go!(self: push_value character), + NotFromSet(ref characters) => go!(self: append_value characters), } }, - //§ attribute-value-(unquoted)-state + // https://html.spec.whatwg.org/#attribute-value-(unquoted)-state states::AttributeValue(Unquoted) => loop { - match pop_except_from!( - self, + // Consume the next input character: + let Some(set_result) = self.pop_except_from( input, - small_char_set!('\r' '\t' '\n' '\x0C' ' ' '&' '>' '\0') - ) { + small_char_set!('\r' '\t' '\n' '\x0C' ' ' '&' '>' '\0'), + ) else { + return ProcessResult::Suspend; + }; + + match set_result { + // ↪ U+0009 CHARACTER TABULATION (tab) + // ↪ U+000A LINE FEED (LF) + // ↪ U+000C FORM FEED (FF) + // ↪ U+0020 SPACE FromSet('\t') | FromSet('\n') | FromSet('\x0C') | FromSet(' ') => { - go!(self: to BeforeAttributeName) + // Switch to the before attribute name state. + go!(self: to State::BeforeAttributeName) + }, + // ↪ U+0026 AMPERSAND (&) + FromSet('&') => { + // Set the return state to the attribute value (unquoted) state. + // Switch to the character reference state. + go!(self: consume_char_ref) + }, + // ↪ U+003E GREATER-THAN SIGN (>) + FromSet('>') => { + // Switch to the data state. + // Emit the current tag token. + go!(self: emit_tag Data) }, - FromSet('&') => go!(self: consume_char_ref), - FromSet('>') => go!(self: emit_tag Data), + // ↪ U+0000 NULL FromSet('\0') => { + // This is an unexpected-null-character parse error. + // Append a U+FFFD REPLACEMENT CHARACTER character to the current attribute's value. self.bad_char_error(); go!(self: push_value '\u{fffd}') }, FromSet(c) => { + // ↪ U+0022 QUOTATION MARK (") + // ↪ U+0027 APOSTROPHE (') + // ↪ U+003C LESS-THAN SIGN (<) + // ↪ U+003D EQUALS SIGN (=) + // ↪ U+0060 GRAVE ACCENT (`) if matches!(c, '"' | '\'' | '<' | '=' | '`') { + // This is an unexpected-character-in-unquoted-attribute-value parse error. + // Treat it as per the "anything else" entry below. self.bad_char_error(); } + // ↪ Anything else + // Append the current input character to the current attribute's value. go!(self: push_value c); }, - NotFromSet(ref b) => go!(self: append_value b), + // ↪ Anything else + NotFromSet(ref characters) => { + // Append the current input character to the current attribute's value. + go!(self: append_value characters) + }, } }, - //§ after-attribute-value-(quoted)-state + // https://html.spec.whatwg.org/#after-attribute-value-(quoted)-state states::AfterAttributeValueQuoted => loop { + // Consume the next input character: match get_char!(self, input) { - '\t' | '\n' | '\x0C' | ' ' => go!(self: to BeforeAttributeName), - '/' => go!(self: to SelfClosingStartTag), - '>' => go!(self: emit_tag Data), + // ↪ U+0009 CHARACTER TABULATION (tab) + // ↪ U+000A LINE FEED (LF) + // ↪ U+000C FORM FEED (FF) + // ↪ U+0020 SPACE + '\t' | '\n' | '\x0C' | ' ' => { + // Switch to the before attribute name state. + go!(self: to State::BeforeAttributeName) + }, + // ↪ U+002F SOLIDUS (/) + '/' => { + // Switch to the self-closing start tag state. + go!(self: to State::SelfClosingStartTag) + }, + // ↪ U+003E GREATER-THAN SIGN (>) + '>' => { + // Switch to the data state. Emit the current tag token. + go!(self: emit_tag Data) + }, + // ↪ Anything else _ => { + // This is a missing-whitespace-between-attributes parse error. + // Reconsume in the before attribute name state. self.bad_char_error(); go!(self: reconsume BeforeAttributeName) }, } }, - //§ self-closing-start-tag-state + // https://html.spec.whatwg.org/#self-closing-start-tag-state states::SelfClosingStartTag => loop { + // Consume the next input character: match get_char!(self, input) { + // ↪ U+003E GREATER-THAN SIGN (>) '>' => { + // Set the self-closing flag of the current tag token. + // Switch to the data state. Emit the current tag token. self.current_tag_self_closing.set(true); go!(self: emit_tag Data); }, + // ↪ Anything else _ => { + // This is an unexpected-solidus-in-tag parse error. + // Reconsume in the before attribute name state. self.bad_char_error(); go!(self: reconsume BeforeAttributeName) }, } }, - //§ comment-start-state - states::CommentStart => loop { + //§ bogus-comment-state + states::BogusComment => loop { + // Consume the next input character: match get_char!(self, input) { - '-' => go!(self: to CommentStartDash), + // ↪ U+003E GREATER-THAN SIGN (>) + '>' => { + // Switch to the data state. Emit the current comment token. + go!(self: emit_comment; to State::Data) + }, + // ↪ U+0000 NULL '\0' => { + // This is an unexpected-null-character parse error. + // Append a U+FFFD REPLACEMENT CHARACTER character to the comment token's data. self.bad_char_error(); - go!(self: push_comment '\u{fffd}'; to Comment) + go!(self: push_comment '\u{fffd}') + }, + // ↪ Anything else + character => { + // Append the current input character to the comment token's data. + go!(self: push_comment character) }, + } + }, + + // https://html.spec.whatwg.org/#markup-declaration-open-state + states::MarkupDeclarationOpen => loop { + // If the next few characters are: + // ↪ Two U+002D HYPHEN-MINUS characters (-) + if eat_exact!(self, input, "--") { + go!(self: clear_comment; to State::CommentStart); + } + // ↪ ASCII case-insensitive match for "DOCTYPE" + else if eat!(self, input, "doctype") { + go!(self: to State::Doctype); + } else { + // ↪ "[CDATA[" + if self + .sink + .adjusted_current_node_present_but_not_in_html_namespace() + && eat_exact!(self, input, "[CDATA[") + { + // Consume those characters. + // If there is an adjusted current node and it is not an element in the HTML namespace, + // then switch to the CDATA section state. Otherwise, this is a cdata-in-html-content parse error. + // Create a comment token whose data is "[CDATA[". Switch to the bogus comment state. + // FIXME: Create that comment token. + go!(self: clear_temp; to State::CdataSection); + } + + // ↪ Anything else + // This is an incorrectly-opened-comment parse error. + // Create a comment token whose data is the empty string. + // Switch to the bogus comment state (don't consume anything in the current state). + // FIXME: Create that comment token. + self.bad_char_error(); + go!(self: clear_comment; to State::BogusComment); + } + }, + + // https://html.spec.whatwg.org/#comment-start-state + states::CommentStart => loop { + // Consume the next input character: + match get_char!(self, input) { + // ↪ U+002D HYPHEN-MINUS (-) + '-' => { + // Switch to the comment start dash state. + go!(self: to State::CommentStartDash) + }, + // ↪ U+003E GREATER-THAN SIGN (>) '>' => { + // This is an abrupt-closing-of-empty-comment parse error. + // Switch to the data state. + // Emit the current comment token. self.bad_char_error(); - go!(self: emit_comment; to Data) + go!(self: emit_comment; to State::Data) }, - c => go!(self: push_comment c; to Comment), + // NOTE: The "anything else" case in the specification reconsumes the character in the + // comment state, instead we inline what the comment state *would* do if it encountered + // that character. + '\0' => { + self.bad_char_error(); + go!(self: push_comment '\u{fffd}'; to State::Comment) + }, + character => go!(self: push_comment character; to State::Comment), } }, - //§ comment-start-dash-state + // https://html.spec.whatwg.org/#comment-start-dash-state states::CommentStartDash => loop { + // Consume the next input character: match get_char!(self, input) { - '-' => go!(self: to CommentEnd), - '\0' => { - self.bad_char_error(); - go!(self: append_comment "-\u{fffd}"; to Comment) + // ↪ U+002D HYPHEN-MINUS (-) + '-' => { + // Switch to the comment end state. + go!(self: to State::CommentEnd) }, + // ↪ U+003E GREATER-THAN SIGN (>) '>' => { + // This is an abrupt-closing-of-empty-comment parse error. + // Switch to the data state. + // Emit the current comment token. + self.bad_char_error(); + go!(self: emit_comment; to State::Data) + }, + // NOTE: The "anything else" case in the specification reconsumes the character in the + // comment state, instead we inline what the comment state *would* do if it encountered + // that character. + '\0' => { self.bad_char_error(); - go!(self: emit_comment; to Data) + go!(self: append_comment "-\u{fffd}"; to State::Comment) + }, + character => { + go!(self: push_comment '-'; push_comment character; to State::Comment) }, - c => go!(self: push_comment '-'; push_comment c; to Comment), } }, - //§ comment-state + // https://html.spec.whatwg.org/#comment-state states::Comment => loop { + // Consume the next input character: match get_char!(self, input) { - c @ '<' => go!(self: push_comment c; to CommentLessThanSign), - '-' => go!(self: to CommentEndDash), + // ↪ U+003C LESS-THAN SIGN (<) + c @ '<' => { + // Append the current input character to the comment token's data. + // Switch to the comment less-than sign state. + go!(self: push_comment c; to State::CommentLessThanSign) + }, + // ↪ U+002D HYPHEN-MINUS (-) + '-' => { + // Switch to the comment end dash state. + go!(self: to State::CommentEndDash) + }, + // ↪ U+0000 NULL '\0' => { + // This is an unexpected-null-character parse error. + // Append a U+FFFD REPLACEMENT CHARACTER character to the comment token's data. self.bad_char_error(); go!(self: push_comment '\u{fffd}') }, - c => go!(self: push_comment c), + // ↪ Anything else + character => { + // Append the current input character to the comment token's data. + go!(self: push_comment character) + }, } }, - //§ comment-less-than-sign-state + // https://html.spec.whatwg.org/#comment-less-than-sign-state states::CommentLessThanSign => loop { + // Consume the next input character: match get_char!(self, input) { - c @ '!' => go!(self: push_comment c; to CommentLessThanSignBang), - c @ '<' => go!(self: push_comment c), - _ => go!(self: reconsume Comment), + // ↪ U+0021 EXCLAMATION MARK (!) + c @ '!' => { + // Append the current input character to the comment token's data. + // Switch to the comment less-than sign bang state. + go!(self: push_comment c; to State::CommentLessThanSignBang) + }, + // ↪ U+003C LESS-THAN SIGN (<) + c @ '<' => { + // Append the current input character to the comment token's data. + go!(self: push_comment c) + }, + // ↪ Anything else + _ => { + // Reconsume in the comment state. + go!(self: reconsume Comment) + }, } }, - //§ comment-less-than-sign-bang + // https://html.spec.whatwg.org/#comment-less-than-sign-bang-state states::CommentLessThanSignBang => loop { + // Consume the next input character: match get_char!(self, input) { - '-' => go!(self: to CommentLessThanSignBangDash), - _ => go!(self: reconsume Comment), + // ↪ U+002D HYPHEN-MINUS (-) + '-' => { + // Switch to the comment less-than sign bang dash state. + go!(self: to State::CommentLessThanSignBangDash) + }, + // ↪ Anything else + _ => { + // Reconsume in the comment state. + go!(self: reconsume Comment) + }, } }, - //§ comment-less-than-sign-bang-dash + // https://html.spec.whatwg.org/#comment-less-than-sign-bang-dash-state states::CommentLessThanSignBangDash => loop { + // Consume the next input character: match get_char!(self, input) { - '-' => go!(self: to CommentLessThanSignBangDashDash), - _ => go!(self: reconsume CommentEndDash), + // ↪ U+002D HYPHEN-MINUS (-) + '-' => { + // Switch to the comment less-than sign bang dash dash state. + go!(self: to State::CommentLessThanSignBangDashDash) + }, + // ↪ Anything else + _ => { + // Reconsume in the comment end dash state. + go!(self: reconsume CommentEndDash) + }, } }, - //§ comment-less-than-sign-bang-dash-dash + // https://html.spec.whatwg.org/#comment-less-than-sign-bang-dash-dash-state states::CommentLessThanSignBangDashDash => loop { match get_char!(self, input) { - '>' => go!(self: reconsume CommentEnd), + // ↪ U+003E GREATER-THAN SIGN (>) + '>' => { + // Reconsume in the comment end state. + go!(self: reconsume CommentEnd) + }, + // ↪ Anything else _ => { + // This is a nested-comment parse error. + // Reconsume in the comment end state. self.bad_char_error(); go!(self: reconsume CommentEnd) }, } }, - //§ comment-end-dash-state + // https://html.spec.whatwg.org/#comment-end-dash-state states::CommentEndDash => loop { + // Consume the next input character: match get_char!(self, input) { - '-' => go!(self: to CommentEnd), + // ↪ U+002D HYPHEN-MINUS (-) + '-' => { + // Switch to the comment end state. + go!(self: to State::CommentEnd) + }, + // NOTE: The "anything else" case in the specification reconsumes the character in the + // comment state. Instead we inline what the comment state *would* do here. '\0' => { self.bad_char_error(); - go!(self: append_comment "-\u{fffd}"; to Comment) + go!(self: append_comment "-\u{fffd}"; to State::Comment) + }, + character => { + go!(self: push_comment '-'; push_comment character; to State::Comment) }, - c => go!(self: push_comment '-'; push_comment c; to Comment), } }, - //§ comment-end-state + // https://html.spec.whatwg.org/#comment-end-state states::CommentEnd => loop { + // Consume the next input character: match get_char!(self, input) { - '>' => go!(self: emit_comment; to Data), - '!' => go!(self: to CommentEndBang), - '-' => go!(self: push_comment '-'), - _ => go!(self: append_comment "--"; reconsume Comment), + // ↪ U+003E GREATER-THAN SIGN (>) + '>' => { + // Switch to the data state. + // Emit the current comment token. + go!(self: emit_comment; to State::Data) + }, + // ↪ U+0021 EXCLAMATION MARK (!) + '!' => { + // Switch to the comment end bang state. + go!(self: to State::CommentEndBang) + }, + // ↪ U+002D HYPHEN-MINUS (-) + '-' => { + // Append a U+002D HYPHEN-MINUS character (-) to the comment token's data. + go!(self: push_comment '-') + }, + // ↪ Anything else + _ => { + // Append two U+002D HYPHEN-MINUS characters (-) to the comment token's data. + // Reconsume in the comment state. + go!(self: append_comment "--"; reconsume Comment) + }, } }, - //§ comment-end-bang-state + // https://html.spec.whatwg.org/#comment-end-bang-state states::CommentEndBang => loop { + // Consume the next input character: match get_char!(self, input) { - '-' => go!(self: append_comment "--!"; to CommentEndDash), + // ↪ U+002D HYPHEN-MINUS (-) + '-' => go!(self: append_comment "--!"; to State::CommentEndDash), + // ↪ U+003E GREATER-THAN SIGN (>) '>' => { self.bad_char_error(); - go!(self: emit_comment; to Data) + go!(self: emit_comment; to State::Data) }, + // NOTE: The "anything else" case in the specification reconsumes the character in the + // comment state. Instead we inline what the comment state *would* do here. '\0' => { self.bad_char_error(); - go!(self: append_comment "--!\u{fffd}"; to Comment) + go!(self: append_comment "--!\u{fffd}"; to State::Comment) + }, + character => { + go!(self: append_comment "--!"; push_comment character; to State::Comment) }, - c => go!(self: append_comment "--!"; push_comment c; to Comment), } }, - //§ doctype-state + // https://html.spec.whatwg.org/#doctype-state states::Doctype => loop { + // Consume the next input character: match get_char!(self, input) { - '\t' | '\n' | '\x0C' | ' ' => go!(self: to BeforeDoctypeName), - '>' => go!(self: reconsume BeforeDoctypeName), + // ↪ U+0009 CHARACTER TABULATION (tab) + // ↪ U+000A LINE FEED (LF) + // ↪ U+000C FORM FEED (FF) + // ↪ U+0020 SPACE + '\t' | '\n' | '\x0C' | ' ' => { + // Switch to the before DOCTYPE name state. + go!(self: to State::BeforeDoctypeName) + }, + // ↪ U+003E GREATER-THAN SIGN (>) + '>' => { + // Reconsume in the before DOCTYPE name state. + go!(self: reconsume BeforeDoctypeName) + }, + // ↪ Anything else _ => { + // This is a missing-whitespace-before-doctype-name parse error. + // Reconsume in the before DOCTYPE name state. self.bad_char_error(); go!(self: reconsume BeforeDoctypeName) }, } }, - //§ before-doctype-name-state + // https://html.spec.whatwg.org/#before-doctype-name-state states::BeforeDoctypeName => loop { + // Consume the next input character: match get_char!(self, input) { - '\t' | '\n' | '\x0C' | ' ' => (), + // ↪ U+0009 CHARACTER TABULATION (tab) + // ↪ U+000A LINE FEED (LF) + // ↪ U+000C FORM FEED (FF) + // ↪ U+0020 SPACE + '\t' | '\n' | '\x0C' | ' ' => { + // Ignore the character. + }, + // ↪ U+0000 NULL '\0' => { + // This is an unexpected-null-character parse error. + // Create a new DOCTYPE token. + // Set the token's name to a U+FFFD REPLACEMENT CHARACTER character. + // Switch to the DOCTYPE name state. self.bad_char_error(); - go!(self: create_doctype; push_doctype_name '\u{fffd}'; to DoctypeName) + go!(self: create_doctype; push_doctype_name '\u{fffd}'; to State::DoctypeName) }, + // ↪ U+003E GREATER-THAN SIGN (>) '>' => { + // This is a missing-doctype-name parse error. + // Create a new DOCTYPE token. + // Set its force-quirks flag to on. + // Switch to the data state. + // Emit the current token. self.bad_char_error(); - go!(self: create_doctype; force_quirks; emit_doctype; to Data) + go!(self: create_doctype; force_quirks; emit_doctype; to State::Data) + }, + // ↪ ASCII upper alpha + // NOTE: This is the same as "anythign else", except we use the lowercase version of the character. + // ↪ Anything else + character => { + // Create a new DOCTYPE token. + // Set the token's name to the current input character. + // Switch to the DOCTYPE name state. + go!(self: create_doctype; push_doctype_name (character.to_ascii_lowercase()); + to State::DoctypeName) }, - c => go!(self: create_doctype; push_doctype_name (c.to_ascii_lowercase()); - to DoctypeName), } }, - //§ doctype-name-state + // https://html.spec.whatwg.org/#doctype-name-state states::DoctypeName => loop { + // Consume the next input character: match get_char!(self, input) { - '\t' | '\n' | '\x0C' | ' ' => go!(self: clear_temp; to AfterDoctypeName), - '>' => go!(self: emit_doctype; to Data), + // ↪ U+0009 CHARACTER TABULATION (tab) + // ↪ U+000A LINE FEED (LF) + // ↪ U+000C FORM FEED (FF) + // ↪ U+0020 SPACE + '\t' | '\n' | '\x0C' | ' ' => { + // Switch to the after DOCTYPE name state. + go!(self: clear_temp; to State::AfterDoctypeName) + }, + // ↪ U+003E GREATER-THAN SIGN (>) + '>' => { + // Switch to the data state. Emit the current DOCTYPE token. + go!(self: emit_doctype; to State::Data) + }, + // ↪ U+0000 NULL '\0' => { + // This is an unexpected-null-character parse error. + // Append a U+FFFD REPLACEMENT CHARACTER character to the current DOCTYPE token's name. self.bad_char_error(); go!(self: push_doctype_name '\u{fffd}') }, - c => go!(self: push_doctype_name (c.to_ascii_lowercase())), + // ↪ ASCII upper alpha + // NOTE: This is the same as "anythign else", except we use the lowercase version of the character. + // ↪ Anything else + character => { + // Append the current input character to the current DOCTYPE token's name. + go!(self: push_doctype_name (character.to_ascii_lowercase())) + }, } }, - //§ after-doctype-name-state + // https://html.spec.whatwg.org/#after-doctype-name-state states::AfterDoctypeName => loop { + // NOTE: We move some steps out of the "anything else" case to the front for convenience. + + // If the six characters starting from the current input character are an + // ASCII case-insensitive match for "PUBLIC", then consume those characters + // and switch to the after DOCTYPE public keyword state. if eat!(self, input, "public") { - go!(self: to AfterDoctypeKeyword Public); - } else if eat!(self, input, "system") { - go!(self: to AfterDoctypeKeyword System); + go!(self: to State::AfterDoctypeKeyword(Public)); + } + // Otherwise, if the six characters starting from the current input character + // are an ASCII case-insensitive match for "SYSTEM", then consume those characters + // and switch to the after DOCTYPE system keyword state. + else if eat!(self, input, "system") { + go!(self: to State::AfterDoctypeKeyword(System)); } else { + // Consume the next input character: match get_char!(self, input) { - '\t' | '\n' | '\x0C' | ' ' => (), - '>' => go!(self: emit_doctype; to Data), + // ↪ U+0009 CHARACTER TABULATION (tab) + // ↪ U+000A LINE FEED (LF) + // ↪ U+000C FORM FEED (FF) + // ↪ U+0020 SPACE + '\t' | '\n' | '\x0C' | ' ' => { + // Ignore the character. + }, + // ↪ U+003E GREATER-THAN SIGN (>) + '>' => { + // Switch to the data state. Emit the current DOCTYPE token. + go!(self: emit_doctype; to State::Data) + }, + // ↪ Anything else _ => { + // Otherwise, this is an invalid-character-sequence-after-doctype-name parse error. + // Set the current DOCTYPE token's force-quirks flag to on. + // Reconsume in the bogus DOCTYPE state. self.bad_char_error(); go!(self: force_quirks; reconsume BogusDoctype) }, @@ -1493,201 +2296,365 @@ impl Tokenizer { } }, - //§ after-doctype-public-keyword-state after-doctype-system-keyword-state + // https://html.spec.whatwg.org/#after-doctype-public-keyword-state + // https://html.spec.whatwg.org/#after-doctype-system-keyword-state states::AfterDoctypeKeyword(kind) => loop { match get_char!(self, input) { - '\t' | '\n' | '\x0C' | ' ' => go!(self: to BeforeDoctypeIdentifier kind), + // ↪ U+0009 CHARACTER TABULATION (tab) + // ↪ U+000A LINE FEED (LF) + // ↪ U+000C FORM FEED (FF) + // ↪ U+0020 SPACE + '\t' | '\n' | '\x0C' | ' ' => { + // Switch to the before DOCTYPE public/system identifier state. + go!(self: to State::BeforeDoctypeIdentifier(kind)) + }, + // ↪ U+0022 QUOTATION MARK (") '"' => { + // This is a missing-whitespace-after-doctype-public/system-keyword parse error. + // Set the current DOCTYPE token's public/system identifier to the empty string + // (not missing), then switch to the DOCTYPE public/system identifier (double-quoted) + // state. self.bad_char_error(); - go!(self: clear_doctype_id kind; to DoctypeIdentifierDoubleQuoted kind) + go!(self: clear_doctype_id kind; to State::DoctypeIdentifierDoubleQuoted(kind)) }, + // ↪ U+0027 APOSTROPHE (') '\'' => { + // This is a missing-whitespace-after-doctype-public-keyword parse error. + // Set the current DOCTYPE token's public identifier to the empty string + // (not missing), then switch to the DOCTYPE public/system identifier (single-quoted) + // state. self.bad_char_error(); - go!(self: clear_doctype_id kind; to DoctypeIdentifierSingleQuoted kind) + go!(self: clear_doctype_id kind; to State::DoctypeIdentifierSingleQuoted(kind)) }, + // ↪ U+003E GREATER-THAN SIGN (>) '>' => { + // This is a missing-doctype-public-identifier parse error. + // Set the current DOCTYPE token's force-quirks flag to on. + // Switch to the data state. + // Emit the current DOCTYPE token. self.bad_char_error(); - go!(self: force_quirks; emit_doctype; to Data) + go!(self: force_quirks; emit_doctype; to State::Data) }, + // ↪ Anything else _ => { + // This is a missing-quote-before-doctype-public-identifier parse error. + // Set the current DOCTYPE token's force-quirks flag to on. + // Reconsume in the bogus DOCTYPE state. self.bad_char_error(); go!(self: force_quirks; reconsume BogusDoctype) }, } }, - //§ before-doctype-public-identifier-state before-doctype-system-identifier-state + // https://html.spec.whatwg.org/#before-doctype-public-identifier-state + // https://html.spec.whatwg.org/#before-doctype-system-identifier-state states::BeforeDoctypeIdentifier(kind) => loop { + // Consume the next input character: match get_char!(self, input) { - '\t' | '\n' | '\x0C' | ' ' => (), - '"' => go!(self: clear_doctype_id kind; to DoctypeIdentifierDoubleQuoted kind), - '\'' => go!(self: clear_doctype_id kind; to DoctypeIdentifierSingleQuoted kind), + // ↪ U+0009 CHARACTER TABULATION (tab) + // ↪ U+000A LINE FEED (LF) + // ↪ U+000C FORM FEED (FF) + // ↪ U+0020 SPACE + '\t' | '\n' | '\x0C' | ' ' => { + // Ignore the character. + }, + // ↪ U+0022 QUOTATION MARK (") + '"' => { + // Set the current DOCTYPE token's public/system identifier to the empty string + // (not missing), then switch to the DOCTYPE public/system identifier (double-quoted) + // state. + go!(self: clear_doctype_id kind; to State::DoctypeIdentifierDoubleQuoted(kind)) + }, + // ↪ U+0027 APOSTROPHE (') + '\'' => { + // Set the current DOCTYPE token's public/systen identifier to the empty string + // (not missing), then switch to the DOCTYPE public/system identifier (single-quoted) + // state. + go!(self: clear_doctype_id kind; to State::DoctypeIdentifierSingleQuoted(kind)) + }, + // ↪ U+003E GREATER-THAN SIGN (>) '>' => { + // This is a missing-doctype-public/system-identifier parse error. + // Set the current DOCTYPE token's force-quirks flag to on. + // Switch to the data state. + // Emit the current DOCTYPE token. self.bad_char_error(); - go!(self: force_quirks; emit_doctype; to Data) + go!(self: force_quirks; emit_doctype; to State::Data) }, + // ↪ Anything else _ => { + // This is a missing-quote-before-doctype-public/system-identifier parse error. + // Set the current DOCTYPE token's force-quirks flag to on. + // Reconsume in the bogus DOCTYPE state. self.bad_char_error(); go!(self: force_quirks; reconsume BogusDoctype) }, } }, - //§ doctype-public-identifier-(double-quoted)-state doctype-system-identifier-(double-quoted)-state + // https://html.spec.whatwg.org/#doctype-public-identifier-(double-quoted)-state + // https://html.spec.whatwg.org/#doctype-system-identifier-(double-quoted)-state states::DoctypeIdentifierDoubleQuoted(kind) => loop { + // Consume the next input character: match get_char!(self, input) { - '"' => go!(self: to AfterDoctypeIdentifier kind), + // ↪ U+0022 QUOTATION MARK (") + '"' => { + // Switch to the after DOCTYPE public/system identifier state. + go!(self: to State::AfterDoctypeIdentifier(kind)) + }, + // ↪ U+0000 NULL '\0' => { + // This is an unexpected-null-character parse error. + // Append a U+FFFD REPLACEMENT CHARACTER character to the current + // DOCTYPE token's public identifier. self.bad_char_error(); go!(self: push_doctype_id kind '\u{fffd}') }, + // ↪ U+003E GREATER-THAN SIGN (>) '>' => { + // This is an abrupt-doctype-public-identifier parse error. + // Set the current DOCTYPE token's force-quirks flag to on. + // Switch to the data state. + // Emit the current DOCTYPE token. self.bad_char_error(); - go!(self: force_quirks; emit_doctype; to Data) + go!(self: force_quirks; emit_doctype; to State::Data) }, - c => go!(self: push_doctype_id kind c), + // ↪ Anything else + character => go!(self: push_doctype_id kind character), } }, - //§ doctype-public-identifier-(single-quoted)-state doctype-system-identifier-(single-quoted)-state + // https://html.spec.whatwg.org/#doctype-public-identifier-(single-quoted)-state + // https://html.spec.whatwg.org/#doctype-system-identifier-(single-quoted)-state states::DoctypeIdentifierSingleQuoted(kind) => loop { + // Consume the next input character: match get_char!(self, input) { - '\'' => go!(self: to AfterDoctypeIdentifier kind), + // ↪ U+0027 APOSTROPHE (') + '\'' => { + // Switch to the after DOCTYPE public/system identifier state. + go!(self: to State::AfterDoctypeIdentifier(kind)) + }, + // ↪ U+0000 NULL '\0' => { + // This is an unexpected-null-character parse error. + // Append a U+FFFD REPLACEMENT CHARACTER character + // to the current DOCTYPE token's public/system identifier. self.bad_char_error(); go!(self: push_doctype_id kind '\u{fffd}') }, + // ↪ U+003E GREATER-THAN SIGN (>) '>' => { + // This is an abrupt-doctype-public/system-identifier parse error. + // Set the current DOCTYPE token's force-quirks flag to on. + // Switch to the data state. + // Emit the current DOCTYPE token. self.bad_char_error(); - go!(self: force_quirks; emit_doctype; to Data) + go!(self: force_quirks; emit_doctype; to State::Data) + }, + // ↪ Anything else + character => { + // Append the current input character to the current DOCTYPE token's + // public/system identifier. + go!(self: push_doctype_id kind character) }, - c => go!(self: push_doctype_id kind c), } }, - //§ after-doctype-public-identifier-state + // https://html.spec.whatwg.org/#after-doctype-public-identifier-state states::AfterDoctypeIdentifier(Public) => loop { + // Consume the next input character: match get_char!(self, input) { + // ↪ U+0009 CHARACTER TABULATION (tab) + // ↪ U+000A LINE FEED (LF) + // ↪ U+000C FORM FEED (FF) + // ↪ U+0020 SPACE '\t' | '\n' | '\x0C' | ' ' => { - go!(self: to BetweenDoctypePublicAndSystemIdentifiers) + // Switch to the between DOCTYPE public and system identifiers state. + go!(self: to State::BetweenDoctypePublicAndSystemIdentifiers) }, - '>' => go!(self: emit_doctype; to Data), + // ↪ U+003E GREATER-THAN SIGN (>) + '>' => { + // Switch to the data state. Emit the current DOCTYPE token. + go!(self: emit_doctype; to State::Data) + }, + // ↪ U+0022 QUOTATION MARK (") '"' => { + // This is a missing-whitespace-between-doctype-public-and-system-identifiers + // parse error. Set the current DOCTYPE token's system identifier to the empty string + // (not missing), then switch to the DOCTYPE system identifier (double-quoted) state. self.bad_char_error(); - go!(self: clear_doctype_id System; to DoctypeIdentifierDoubleQuoted System) + go!(self: clear_doctype_id System; to State::DoctypeIdentifierDoubleQuoted(System)) }, + // ↪ U+0027 APOSTROPHE (') '\'' => { + // This is a missing-whitespace-between-doctype-public-and-system-identifiers parse error. + // Set the current DOCTYPE token's system identifier to the empty string (not missing), + // then switch to the DOCTYPE system identifier (single-quoted) state. self.bad_char_error(); - go!(self: clear_doctype_id System; to DoctypeIdentifierSingleQuoted System) + go!(self: clear_doctype_id System; to State::DoctypeIdentifierSingleQuoted(System)) }, + // ↪ Anything else _ => { + // This is a missing-quote-before-doctype-system-identifier parse error. + // Set the current DOCTYPE token's force-quirks flag to on. + // Reconsume in the bogus DOCTYPE state. self.bad_char_error(); go!(self: force_quirks; reconsume BogusDoctype) }, } }, - //§ after-doctype-system-identifier-state - states::AfterDoctypeIdentifier(System) => loop { - match get_char!(self, input) { - '\t' | '\n' | '\x0C' | ' ' => (), - '>' => go!(self: emit_doctype; to Data), - _ => { - self.bad_char_error(); - go!(self: reconsume BogusDoctype) - }, - } - }, - - //§ between-doctype-public-and-system-identifiers-state + // https://html.spec.whatwg.org/#between-doctype-public-and-system-identifiers-state states::BetweenDoctypePublicAndSystemIdentifiers => loop { + // Consume the next input character: match get_char!(self, input) { - '\t' | '\n' | '\x0C' | ' ' => (), - '>' => go!(self: emit_doctype; to Data), + // ↪ U+0009 CHARACTER TABULATION (tab) + // ↪ U+000A LINE FEED (LF) + // ↪ U+000C FORM FEED (FF) + // ↪ U+0020 SPACE + '\t' | '\n' | '\x0C' | ' ' => { + // Ignore the character. + }, + // ↪ U+003E GREATER-THAN SIGN (>) + '>' => { + // Switch to the data state. Emit the current DOCTYPE token. + go!(self: emit_doctype; to State::Data) + }, + // ↪ U+0022 QUOTATION MARK (") '"' => { - go!(self: clear_doctype_id System; to DoctypeIdentifierDoubleQuoted System) + // Set the current DOCTYPE token's system identifier to the empty string (not missing), + // then switch to the DOCTYPE system identifier (double-quoted) state. + go!(self: clear_doctype_id System; to State::DoctypeIdentifierDoubleQuoted(System)) }, + // ↪ U+0027 APOSTROPHE (') '\'' => { - go!(self: clear_doctype_id System; to DoctypeIdentifierSingleQuoted System) + // Set the current DOCTYPE token's system identifier to the empty string (not missing), + // then switch to the DOCTYPE system identifier (single-quoted) state. + go!(self: clear_doctype_id System; to State::DoctypeIdentifierSingleQuoted(System)) }, + // ↪ Anything else _ => { + // This is a missing-quote-before-doctype-system-identifier parse error. + // Set the current DOCTYPE token's force-quirks flag to on. + // Reconsume in the bogus DOCTYPE state. self.bad_char_error(); go!(self: force_quirks; reconsume BogusDoctype) }, } }, - //§ bogus-doctype-state - states::BogusDoctype => loop { + // https://html.spec.whatwg.org/#after-doctype-system-identifier-state + states::AfterDoctypeIdentifier(System) => loop { + // Consume the next input character: match get_char!(self, input) { - '>' => go!(self: emit_doctype; to Data), - '\0' => { + // ↪ U+0009 CHARACTER TABULATION (tab) + // ↪ U+000A LINE FEED (LF) + // ↪ U+000C FORM FEED (FF) + // ↪ U+0020 SPACE + '\t' | '\n' | '\x0C' | ' ' => { + // Ignore the character. + }, + // ↪ U+003E GREATER-THAN SIGN (>) + '>' => { + // Switch to the data state. Emit the current DOCTYPE token. + go!(self: emit_doctype; to State::Data) + }, + // ↪ Anything else + _ => { + // This is an unexpected-character-after-doctype-system-identifier parse error. + // Reconsume in the bogus DOCTYPE state. self.bad_char_error(); + go!(self: reconsume BogusDoctype) }, - _ => (), } }, - //§ bogus-comment-state - states::BogusComment => loop { + // https://html.spec.whatwg.org/#bogus-doctype-state + states::BogusDoctype => loop { + // Consume the next input character: match get_char!(self, input) { - '>' => go!(self: emit_comment; to Data), + // ↪ U+003E GREATER-THAN SIGN (>) + '>' => { + // Switch to the data state. + // Emit the current DOCTYPE token. + go!(self: emit_doctype; to State::Data) + }, + // ↪ U+0000 NULL '\0' => { + // This is an unexpected-null-character parse error. + // Ignore the character. self.bad_char_error(); - go!(self: push_comment '\u{fffd}') }, - c => go!(self: push_comment c), - } - }, - - //§ markup-declaration-open-state - states::MarkupDeclarationOpen => loop { - if eat_exact!(self, input, "--") { - go!(self: clear_comment; to CommentStart); - } else if eat!(self, input, "doctype") { - go!(self: to Doctype); - } else { - if self - .sink - .adjusted_current_node_present_but_not_in_html_namespace() - && eat_exact!(self, input, "[CDATA[") - { - go!(self: clear_temp; to CdataSection); - } - self.bad_char_error(); - go!(self: clear_comment; to BogusComment); + // ↪ Anything else + _ => { + // Ignore the character. + }, } }, - //§ cdata-section-state + // https://html.spec.whatwg.org/#cdata-section-state states::CdataSection => loop { + // Consume the next input character: match get_char!(self, input) { - ']' => go!(self: to CdataSectionBracket), + // ↪ U+005D RIGHT SQUARE BRACKET (]) + ']' => { + // Switch to the CDATA section bracket state. + go!(self: to State::CdataSectionBracket) + }, + // FIXME: This is not in the specification. '\0' => { self.emit_temp_buf(); self.emit_char('\0'); }, - c => go!(self: push_temp c), + // ↪ Anything else + character => { + // Emit the current input character as a character token. + go!(self: push_temp character) + }, } }, - //§ cdata-section-bracket - states::CdataSectionBracket => match get_char!(self, input) { - ']' => go!(self: to CdataSectionEnd), - _ => go!(self: push_temp ']'; reconsume CdataSection), + // https://html.spec.whatwg.org/#cdata-section-bracket-state + states::CdataSectionBracket => { + // Consume the next input character: + match get_char!(self, input) { + // ↪ U+005D RIGHT SQUARE BRACKET (]) + ']' => { + // Switch to the CDATA section end state. + go!(self: to State::CdataSectionEnd) + }, + // ↪ Anything else + _ => { + // Emit a U+005D RIGHT SQUARE BRACKET character token. Reconsume in the CDATA section state. + go!(self: push_temp ']'; reconsume CdataSection) + }, + } }, - //§ cdata-section-end + // https://html.spec.whatwg.org/#cdata-section-end-state states::CdataSectionEnd => loop { + // Consume the next input character: match get_char!(self, input) { - ']' => go!(self: push_temp ']'), + // U+005D RIGHT SQUARE BRACKET (]) + ']' => { + // Emit a U+005D RIGHT SQUARE BRACKET character token. + go!(self: push_temp ']') + }, + // U+003E GREATER-THAN SIGN (>) '>' => { + // Switch to the data state. self.emit_temp_buf(); - go!(self: to Data); + go!(self: to State::Data); + }, + // Anything else + _ => { + // Emit two U+005D RIGHT SQUARE BRACKET character tokens. + // Reconsume in the CDATA section state. + go!(self: push_temp ']'; push_temp ']'; reconsume CdataSection) }, - _ => go!(self: push_temp ']'; push_temp ']'; reconsume CdataSection), } }, + // TODO: What about the processing-instruction related states? //§ END } } @@ -1774,12 +2741,9 @@ impl Tokenizer { .iter() .map(|(s, t)| (*s, *t)) .collect(); - results.sort_by(|&(_, x), &(_, y)| y.cmp(&x)); + results.sort_by_key(|&(_, x)| Reverse(x)); - let total: u64 = results - .iter() - .map(|&(_, t)| t) - .fold(0, ::std::ops::Add::add); + let total: u64 = results.iter().map(|&(_, t)| t).sum(); println!("\nTokenizer profile, in nanoseconds"); println!( "\n{:12} total in token sink", @@ -1813,52 +2777,54 @@ impl Tokenizer { | states::ScriptDataEscapedDash(_) | states::ScriptDataEscapedDashDash(_) => { self.bad_eof_error(); - go!(self: to Data) + go!(self: to State::Data) }, - states::BeforeAttributeValue => go!(self: reconsume AttributeValue Unquoted), + states::BeforeAttributeValue => go!(self: reconsume AttributeValue(Unquoted)), states::TagOpen => { self.bad_eof_error(); self.emit_char('<'); - go!(self: to Data); + go!(self: to State::Data); }, states::EndTagOpen => { self.bad_eof_error(); self.emit_char('<'); self.emit_char('/'); - go!(self: to Data); + go!(self: to State::Data); }, states::RawLessThanSign(ScriptDataEscaped(DoubleEscaped)) => { - go!(self: to RawData ScriptDataEscaped DoubleEscaped) + go!(self: to State::RawData(ScriptDataEscaped(DoubleEscaped))) }, states::RawLessThanSign(kind) => { self.emit_char('<'); - go!(self: to RawData kind); + go!(self: to State::RawData(kind)); }, states::RawEndTagOpen(kind) => { self.emit_char('<'); self.emit_char('/'); - go!(self: to RawData kind); + go!(self: to State::RawData(kind)); }, states::RawEndTagName(kind) => { self.emit_char('<'); self.emit_char('/'); self.emit_temp_buf(); - go!(self: to RawData kind) + go!(self: to State::RawData(kind)) }, - states::ScriptDataEscapeStart(kind) => go!(self: to RawData ScriptDataEscaped kind), + states::ScriptDataEscapeStart(kind) => { + go!(self: to State::RawData(ScriptDataEscaped(kind))) + }, - states::ScriptDataEscapeStartDash => go!(self: to RawData ScriptData), + states::ScriptDataEscapeStartDash => go!(self: to State::RawData(ScriptData)), states::ScriptDataDoubleEscapeEnd => { - go!(self: to RawData ScriptDataEscaped DoubleEscaped) + go!(self: to State::RawData(ScriptDataEscaped(DoubleEscaped))) }, states::CommentStart @@ -1868,7 +2834,7 @@ impl Tokenizer { | states::CommentEnd | states::CommentEndBang => { self.bad_eof_error(); - go!(self: emit_comment; to Data) + go!(self: emit_comment; to State::Data) }, states::CommentLessThanSign | states::CommentLessThanSignBang => { @@ -1881,7 +2847,7 @@ impl Tokenizer { states::Doctype | states::BeforeDoctypeName => { self.bad_eof_error(); - go!(self: create_doctype; force_quirks; emit_doctype; to Data) + go!(self: create_doctype; force_quirks; emit_doctype; to State::Data) }, states::DoctypeName @@ -1893,27 +2859,29 @@ impl Tokenizer { | states::AfterDoctypeIdentifier(_) | states::BetweenDoctypePublicAndSystemIdentifiers => { self.bad_eof_error(); - go!(self: force_quirks; emit_doctype; to Data) + go!(self: force_quirks; emit_doctype; to State::Data) }, - states::BogusDoctype => go!(self: emit_doctype; to Data), + states::BogusDoctype => go!(self: emit_doctype; to State::Data), - states::BogusComment => go!(self: emit_comment; to Data), + states::BogusComment => go!(self: emit_comment; to State::Data), states::MarkupDeclarationOpen => { self.bad_char_error(); - go!(self: to BogusComment) + go!(self: to State::BogusComment) }, states::CdataSection => { self.emit_temp_buf(); self.bad_eof_error(); - go!(self: to Data) + go!(self: to State::Data) }, - states::CdataSectionBracket => go!(self: push_temp ']'; to CdataSection), + states::CdataSectionBracket => go!(self: push_temp ']'; to State::CdataSection), - states::CdataSectionEnd => go!(self: push_temp ']'; push_temp ']'; to CdataSection), + states::CdataSectionEnd => { + go!(self: push_temp ']'; push_temp ']'; to State::CdataSection) + }, } } diff --git a/vendor/html5ever/src/tree_builder/mod.rs b/vendor/html5ever/src/tree_builder/mod.rs index e58a4c4..9b4b311 100644 --- a/vendor/html5ever/src/tree_builder/mod.rs +++ b/vendor/html5ever/src/tree_builder/mod.rs @@ -1282,10 +1282,8 @@ where _ => continue, }; match *name { - local_name!("td") | local_name!("th") => { - if !last { - return InsertionMode::InCell; - } + local_name!("td") | local_name!("th") if !last => { + return InsertionMode::InCell; }, local_name!("tr") => return InsertionMode::InRow, local_name!("tbody") | local_name!("thead") | local_name!("tfoot") => { @@ -1375,14 +1373,6 @@ where declare_tag_set!(listed = [form_associatable] - "img"); // Step 7. - let qname = QualName::new(None, ns, name); - let elem = create_element_with_flags( - &self.sink, - qname.clone(), - attrs.clone(), - had_duplicate_attributes, - ); - let insertion_point = self.appropriate_place_for_insertion(None); let (node1, node2) = match insertion_point { InsertionPoint::LastChild(ref p) | InsertionPoint::BeforeSibling(ref p) => { @@ -1395,14 +1385,20 @@ where }; // Step 12. - if form_associatable(qname.expanded()) + let qname = QualName::new(None, ns, name); + let form_is_associatable = form_associatable(qname.expanded()) && self.form_elem.borrow().is_some() && !self.in_html_elem_named(local_name!("template")) && !(listed(qname.expanded()) && attrs .iter() - .any(|a| a.name.expanded() == expanded_name!("", "form"))) - { + .any(|a| a.name.expanded() == expanded_name!("", "form"))); + + // By checking whether the form is associatable first, then creating the element + // we can avoid cloning the attributes. + let elem = create_element_with_flags(&self.sink, qname, attrs, had_duplicate_attributes); + + if form_is_associatable { let form = self.form_elem.borrow().as_ref().unwrap().clone(); self.sink .associate_with_form(&elem, &form, (&node1, node2.as_ref())); @@ -1451,12 +1447,8 @@ where ) -> Handle { let adjusted_insertion_location = self.appropriate_place_for_insertion(None); let qname = QualName::new(None, ns, tag.name.clone()); - let elem = create_element_with_flags( - &self.sink, - qname.clone(), - tag.attrs.clone(), - tag.had_duplicate_attributes, - ); + let elem = + create_element_with_flags(&self.sink, qname, tag.attrs, tag.had_duplicate_attributes); if !only_add_to_element_stack { self.insert_at(adjusted_insertion_location, AppendNode(elem.clone()));