From 8845828ca5a17c2f7d753340da8ad1f07c84b9be Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Joubert?= Date: Mon, 26 Oct 2026 20:17:57 +0300 Subject: [PATCH 1/2] Copy verbatim runs in the plaintext renderer Bumps cmarkly to the commit that lets S_out copy runs of characters the renderer writes back unchanged (LITERAL escaping, or the plaintext renderer whose outc is the identity) with a single put, instead of decoding, re-encoding and appending each code point. Spaces when wrapping, newlines in LITERAL, invalid UTF-8 and extensions with a commonmark_escape_func still take the per-character path, and a run never starts once a break opportunity is pending, so wrapping is unchanged. Co-Authored-By: Claude Opus 5.5 --- cmarkly | 2 +- ext/markly/plaintext.c | 8 +---- ext/markly/render.c | 51 +++++++++++++++++++++++++++++++- ext/markly/render.h | 3 ++ releases.md | 4 +++ test/markly/node/to_plaintext.rb | 29 ++++++++++++++++++ 6 files changed, 88 insertions(+), 9 deletions(-) diff --git a/cmarkly b/cmarkly index 85ef5cf..bed0622 160000 --- a/cmarkly +++ b/cmarkly @@ -1 +1 @@ -Subproject commit 85ef5cf05aa13000bb3a7c9579ac6091341f814a +Subproject commit bed0622cfb6db283108cec2befcb3149ece710b6 diff --git a/ext/markly/plaintext.c b/ext/markly/plaintext.c index e212b19..0ab3f0f 100644 --- a/ext/markly/plaintext.c +++ b/ext/markly/plaintext.c @@ -10,12 +10,6 @@ // Functions to convert cmark_nodes to plain text strings. -static CMARK_INLINE void outc(cmark_renderer *renderer, cmark_node *node, - cmark_escaping escape, - int32_t c, unsigned char nextc) { - cmark_render_code_point(renderer, c); -} - static int S_render_node(cmark_renderer *renderer, cmark_node *node, cmark_event_type ev_type, int options) { int list_number; @@ -217,5 +211,5 @@ char *cmark_render_plaintext_with_mem(cmark_node *root, int options, int width, // a different meaning with OPT_HARDBREAKS width = 0; } - return cmark_render(mem, root, options, width, outc, S_render_node); + return cmark_render(mem, root, options, width, NULL, S_render_node); } diff --git a/ext/markly/render.c b/ext/markly/render.c index 20916fe..28483a7 100644 --- a/ext/markly/render.c +++ b/ext/markly/render.c @@ -57,6 +57,11 @@ static void S_out(cmark_renderer *renderer, cmark_node *node, renderer->need_cr -= 1; } + // Characters the renderer writes back unchanged are copied in runs rather + // than decoded and re-encoded one by one. Spaces (when wrapping), newlines + // (in LITERAL) and invalid UTF-8 still go through the loop below. + bool verbatim = !ext && (escape == LITERAL || renderer->outc_verbatim); + while (i < length) { if (renderer->begin_line) { cmark_strbuf_put(renderer->buffer, renderer->prefix->ptr, @@ -65,6 +70,40 @@ static void S_out(cmark_renderer *renderer, cmark_node *node, renderer->column = renderer->prefix->size; } + // A run cannot cross a line-break opportunity: last_breakable stays 0. + if (verbatim && (renderer->width == 0 || renderer->last_breakable == 0)) { + int run_end = i; + int code_points = 0; + bool all_digits = true; + while (run_end < length) { + unsigned char byte = (unsigned char)source[run_end]; + if (byte < 0x80) { + if ((byte == ' ' && wrap) || (byte == '\n' && escape == LITERAL)) + break; + all_digits = all_digits && cmark_isdigit((char)byte); + run_end += 1; + } else { + len = cmark_utf8proc_iterate((const uint8_t *)source + run_end, + length - run_end, &c); + if (len == -1) + break; + // (char)c truncates, as the per-character path below does. + all_digits = all_digits && cmark_isdigit((char)c); + run_end += len; + } + code_points += 1; + } + if (run_end > i) { + cmark_strbuf_put(renderer->buffer, (const unsigned char *)source + i, + run_end - i); + renderer->column += code_points; + renderer->begin_line = false; + renderer->begin_content = renderer->begin_content && all_digits; + i = run_end; + continue; + } + } + len = cmark_utf8proc_iterate((const uint8_t *)source + i, length - i, &c); if (len == -1) { // error condition return; // return without rendering rest of string @@ -155,6 +194,12 @@ void cmark_render_code_point(cmark_renderer *renderer, uint32_t c) { renderer->column += 1; } +static void S_outc_verbatim(cmark_renderer *renderer, cmark_node *node, + cmark_escaping escape, int32_t c, + unsigned char nextc) { + cmark_render_code_point(renderer, c); +} + char *cmark_render(cmark_mem *mem, cmark_node *root, int options, int width, void (*outc)(cmark_renderer *, cmark_node *, cmark_escaping, int32_t, @@ -169,10 +214,14 @@ char *cmark_render(cmark_mem *mem, cmark_node *root, int options, int width, char *result; cmark_iter *iter = cmark_iter_new(root); + bool outc_verbatim = outc == NULL; + if (outc_verbatim) + outc = S_outc_verbatim; + cmark_renderer renderer = {mem, &buf, &pref, 0, width, 0, 0, true, true, false, false, outc, S_cr, S_blankline, S_out, - 0}; + 0, outc_verbatim}; while ((ev_type = cmark_iter_next(iter)) != CMARK_EVENT_DONE) { cur = cmark_iter_get_node(iter); diff --git a/ext/markly/render.h b/ext/markly/render.h index 4a68d1e..655a097 100644 --- a/ext/markly/render.h +++ b/ext/markly/render.h @@ -28,6 +28,7 @@ struct cmark_renderer { void (*blankline)(struct cmark_renderer *); void (*out)(struct cmark_renderer *, cmark_node *, const char *, bool, cmark_escaping); unsigned int footnote_ix; + bool outc_verbatim; }; typedef struct cmark_renderer cmark_renderer; @@ -47,6 +48,8 @@ void cmark_render_ascii(cmark_renderer *renderer, const char *s); void cmark_render_code_point(cmark_renderer *renderer, uint32_t c); +// A NULL outc writes every code point unchanged, which lets S_out copy runs. + char *cmark_render(cmark_mem *mem, cmark_node *root, int options, int width, void (*outc)(cmark_renderer *, cmark_node *, cmark_escaping, int32_t, diff --git a/releases.md b/releases.md index a6599f2..8fd99d3 100644 --- a/releases.md +++ b/releases.md @@ -1,5 +1,9 @@ # Releases +## Unreleased + + - Speed up `Node#to_plaintext` by copying runs of unchanged text in one append instead of decoding and re-encoding each character (`parse` + `to_plaintext` 1.3 to 1.7 times faster). + ## v0.19.0 - Add `Markly::HTML_BLOCK_BLANK_LINES` for keeping consistently indented HTML content together across blank lines. diff --git a/test/markly/node/to_plaintext.rb b/test/markly/node/to_plaintext.rb index 95b3751..0542b23 100644 --- a/test/markly/node/to_plaintext.rb +++ b/test/markly/node/to_plaintext.rb @@ -44,4 +44,33 @@ it "can generate plaintext" do expect(document.to_plaintext).to be == PLAINTEXT end + + with "text the renderer copies in runs" do + it "wraps at the width inside a list item" do + document = Markly.parse("- Les lunettes Xpand X105-RF-X1 sont des lunettes 3D actives.") + + expect(document.to_plaintext(width: 20)).to be == " - Les lunettes\n Xpand X105-RF-X1\n sont des\n lunettes 3D\n actives.\n" + end + + it "keeps multibyte characters intact" do + document = Markly.parse("> Déjà vu: l'été à Zürich 日本 😀") + + expect(document.to_plaintext).to be == "Déjà vu: l'été à Zürich 日本 😀\n" + end + + it "indents every line of a code block in a list item" do + document = Markly.parse("1. item\n\n ```\n code line one\n code line two\n ```") + + expect(document.to_plaintext).to be == "1. item\n \n code line one\n code line two\n" + end + + it "stops at invalid UTF-8 in a text node built through the API" do + document = Markly.parse("- item") + text = Markly::Node.new(:text) + text.string_content = "valid \xFF lost".b.force_encoding(Encoding::UTF_8) + document.first_child.first_child.first_child.append_child(text) + + expect(document.to_plaintext).to be == " - itemvalid \n" + end + end end From 0dbe62896f8198e68d2a5816b3bfc5839e2556c6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Joubert?= Date: Tue, 27 Oct 2026 12:17:54 +0300 Subject: [PATCH 2/2] Keep U+FFFE and U+FFFF out of verbatim runs Bumps cmarkly to the commit that stops runs on these two code points, which cmark_utf8proc_encode_char writes as a single byte. Co-Authored-By: Claude Opus 5.5 --- cmarkly | 2 +- ext/markly/render.c | 42 ++++++++++++++++++++++-------------------- ext/markly/render.h | 2 -- 3 files changed, 23 insertions(+), 23 deletions(-) diff --git a/cmarkly b/cmarkly index bed0622..23f5ae2 160000 --- a/cmarkly +++ b/cmarkly @@ -1 +1 @@ -Subproject commit bed0622cfb6db283108cec2befcb3149ece710b6 +Subproject commit 23f5ae2bfa456cf49ac7e232a90648c1e83387f3 diff --git a/ext/markly/render.c b/ext/markly/render.c index 28483a7..9d97def 100644 --- a/ext/markly/render.c +++ b/ext/markly/render.c @@ -19,6 +19,12 @@ static CMARK_INLINE void S_blankline(cmark_renderer *renderer) { } } +static void S_outc_verbatim(cmark_renderer *renderer, cmark_node *node, + cmark_escaping escape, int32_t c, + unsigned char nextc) { + cmark_render_code_point(renderer, c); +} + static void S_out(cmark_renderer *renderer, cmark_node *node, const char *source, bool wrap, cmark_escaping escape) { @@ -59,8 +65,10 @@ static void S_out(cmark_renderer *renderer, cmark_node *node, // Characters the renderer writes back unchanged are copied in runs rather // than decoded and re-encoded one by one. Spaces (when wrapping), newlines - // (in LITERAL) and invalid UTF-8 still go through the loop below. - bool verbatim = !ext && (escape == LITERAL || renderer->outc_verbatim); + // (in LITERAL), U+FFFE, U+FFFF and invalid UTF-8 still go through the loop + // below. + bool verbatim = + !ext && (escape == LITERAL || renderer->outc == S_outc_verbatim); while (i < length) { if (renderer->begin_line) { @@ -70,7 +78,7 @@ static void S_out(cmark_renderer *renderer, cmark_node *node, renderer->column = renderer->prefix->size; } - // A run cannot cross a line-break opportunity: last_breakable stays 0. + // With no pending break point, the width check below cannot fire. if (verbatim && (renderer->width == 0 || renderer->last_breakable == 0)) { int run_end = i; int code_points = 0; @@ -78,19 +86,20 @@ static void S_out(cmark_renderer *renderer, cmark_node *node, while (run_end < length) { unsigned char byte = (unsigned char)source[run_end]; if (byte < 0x80) { - if ((byte == ' ' && wrap) || (byte == '\n' && escape == LITERAL)) - break; - all_digits = all_digits && cmark_isdigit((char)byte); - run_end += 1; + c = byte; + len = 1; } else { len = cmark_utf8proc_iterate((const uint8_t *)source + run_end, length - run_end, &c); - if (len == -1) + // cmark_utf8proc_encode_char writes U+FFFE and U+FFFF as one byte. + if (len == -1 || c == 0xFFFE || c == 0xFFFF) break; - // (char)c truncates, as the per-character path below does. - all_digits = all_digits && cmark_isdigit((char)c); - run_end += len; } + if ((c == ' ' && wrap) || (c == '\n' && escape == LITERAL)) + break; + // (char)c truncates, as the per-character path below does. + all_digits = all_digits && cmark_isdigit((char)c); + run_end += len; code_points += 1; } if (run_end > i) { @@ -194,12 +203,6 @@ void cmark_render_code_point(cmark_renderer *renderer, uint32_t c) { renderer->column += 1; } -static void S_outc_verbatim(cmark_renderer *renderer, cmark_node *node, - cmark_escaping escape, int32_t c, - unsigned char nextc) { - cmark_render_code_point(renderer, c); -} - char *cmark_render(cmark_mem *mem, cmark_node *root, int options, int width, void (*outc)(cmark_renderer *, cmark_node *, cmark_escaping, int32_t, @@ -214,14 +217,13 @@ char *cmark_render(cmark_mem *mem, cmark_node *root, int options, int width, char *result; cmark_iter *iter = cmark_iter_new(root); - bool outc_verbatim = outc == NULL; - if (outc_verbatim) + if (outc == NULL) outc = S_outc_verbatim; cmark_renderer renderer = {mem, &buf, &pref, 0, width, 0, 0, true, true, false, false, outc, S_cr, S_blankline, S_out, - 0, outc_verbatim}; + 0}; while ((ev_type = cmark_iter_next(iter)) != CMARK_EVENT_DONE) { cur = cmark_iter_get_node(iter); diff --git a/ext/markly/render.h b/ext/markly/render.h index 655a097..0c6c079 100644 --- a/ext/markly/render.h +++ b/ext/markly/render.h @@ -28,7 +28,6 @@ struct cmark_renderer { void (*blankline)(struct cmark_renderer *); void (*out)(struct cmark_renderer *, cmark_node *, const char *, bool, cmark_escaping); unsigned int footnote_ix; - bool outc_verbatim; }; typedef struct cmark_renderer cmark_renderer; @@ -49,7 +48,6 @@ void cmark_render_ascii(cmark_renderer *renderer, const char *s); void cmark_render_code_point(cmark_renderer *renderer, uint32_t c); // A NULL outc writes every code point unchanged, which lets S_out copy runs. - char *cmark_render(cmark_mem *mem, cmark_node *root, int options, int width, void (*outc)(cmark_renderer *, cmark_node *, cmark_escaping, int32_t,