From bed0622cfb6db283108cec2befcb3149ece710b6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Joubert?= Date: Mon, 26 Oct 2026 20:14:53 +0300 Subject: [PATCH 1/2] Copy verbatim runs in the plaintext renderer S_out decoded every code point, re-encoded it and appended it to the buffer one at a time, even when the renderer writes it back unchanged (LITERAL escaping, or the plaintext renderer whose outc is the identity). Runs of such characters are now validated and copied with a single put. Spaces when wrapping, newlines in LITERAL, invalid UTF-8 and extensions with a commonmark_escape_func still take the per-character path, and a run never starts once a break opportunity is pending, so wrapping is unchanged. Co-Authored-By: Claude Opus 5.5 --- api_test/main.c | 24 +++++++++++++++++++++++ src/plaintext.c | 8 +------- src/render.c | 51 ++++++++++++++++++++++++++++++++++++++++++++++++- src/render.h | 3 +++ 4 files changed, 78 insertions(+), 8 deletions(-) diff --git a/api_test/main.c b/api_test/main.c index 0b4d1c4db..78a139797 100644 --- a/api_test/main.c +++ b/api_test/main.c @@ -995,6 +995,30 @@ static void render_plaintext(test_batch_runner *runner) { "render document without wrapping"); free(plaintext); + static const char multibyte_markdown[] = + "- D\xC3\xA9j\xC3\xA0 vu \xC3\xA0 Z\xC3\xBCrich, \xE6\x97\xA5\xE6\x9C\xAC " + "\xF0\x9F\x98\x80 encore\n" + "\n" + " ```\n" + " code one\n" + " code two\n" + " ```\n"; + cmark_node *multibyte_doc = cmark_parse_document( + multibyte_markdown, sizeof(multibyte_markdown) - 1, CMARK_OPT_DEFAULT); + plaintext = cmark_render_plaintext(multibyte_doc, CMARK_OPT_DEFAULT, 12); + STR_EQ(runner, plaintext, + " - D\xC3\xA9j\xC3\xA0 vu\n" + " \xC3\xA0\n" + " Z\xC3\xBCrich,\n" + " \xE6\x97\xA5\xE6\x9C\xAC \xF0\x9F\x98\x80\n" + " encore\n" + " \n" + " code one\n" + " code two\n", + "render multibyte text and code block with wrapping"); + free(plaintext); + cmark_node_free(multibyte_doc); + cmark_node *text = cmark_node_new(CMARK_NODE_TEXT); cmark_node_set_literal(text, "Hi"); plaintext = cmark_render_plaintext(text, CMARK_OPT_DEFAULT, 0); diff --git a/src/plaintext.c b/src/plaintext.c index e212b19b9..0ab3f0f3b 100644 --- a/src/plaintext.c +++ b/src/plaintext.c @@ -10,12 +10,6 @@ // Functions to convert cmark_nodes to plain text strings. -static CMARK_INLINE void outc(cmark_renderer *renderer, cmark_node *node, - cmark_escaping escape, - int32_t c, unsigned char nextc) { - cmark_render_code_point(renderer, c); -} - static int S_render_node(cmark_renderer *renderer, cmark_node *node, cmark_event_type ev_type, int options) { int list_number; @@ -217,5 +211,5 @@ char *cmark_render_plaintext_with_mem(cmark_node *root, int options, int width, // a different meaning with OPT_HARDBREAKS width = 0; } - return cmark_render(mem, root, options, width, outc, S_render_node); + return cmark_render(mem, root, options, width, NULL, S_render_node); } diff --git a/src/render.c b/src/render.c index 20916feb9..28483a762 100644 --- a/src/render.c +++ b/src/render.c @@ -57,6 +57,11 @@ static void S_out(cmark_renderer *renderer, cmark_node *node, renderer->need_cr -= 1; } + // Characters the renderer writes back unchanged are copied in runs rather + // than decoded and re-encoded one by one. Spaces (when wrapping), newlines + // (in LITERAL) and invalid UTF-8 still go through the loop below. + bool verbatim = !ext && (escape == LITERAL || renderer->outc_verbatim); + while (i < length) { if (renderer->begin_line) { cmark_strbuf_put(renderer->buffer, renderer->prefix->ptr, @@ -65,6 +70,40 @@ static void S_out(cmark_renderer *renderer, cmark_node *node, renderer->column = renderer->prefix->size; } + // A run cannot cross a line-break opportunity: last_breakable stays 0. + if (verbatim && (renderer->width == 0 || renderer->last_breakable == 0)) { + int run_end = i; + int code_points = 0; + bool all_digits = true; + while (run_end < length) { + unsigned char byte = (unsigned char)source[run_end]; + if (byte < 0x80) { + if ((byte == ' ' && wrap) || (byte == '\n' && escape == LITERAL)) + break; + all_digits = all_digits && cmark_isdigit((char)byte); + run_end += 1; + } else { + len = cmark_utf8proc_iterate((const uint8_t *)source + run_end, + length - run_end, &c); + if (len == -1) + break; + // (char)c truncates, as the per-character path below does. + all_digits = all_digits && cmark_isdigit((char)c); + run_end += len; + } + code_points += 1; + } + if (run_end > i) { + cmark_strbuf_put(renderer->buffer, (const unsigned char *)source + i, + run_end - i); + renderer->column += code_points; + renderer->begin_line = false; + renderer->begin_content = renderer->begin_content && all_digits; + i = run_end; + continue; + } + } + len = cmark_utf8proc_iterate((const uint8_t *)source + i, length - i, &c); if (len == -1) { // error condition return; // return without rendering rest of string @@ -155,6 +194,12 @@ void cmark_render_code_point(cmark_renderer *renderer, uint32_t c) { renderer->column += 1; } +static void S_outc_verbatim(cmark_renderer *renderer, cmark_node *node, + cmark_escaping escape, int32_t c, + unsigned char nextc) { + cmark_render_code_point(renderer, c); +} + char *cmark_render(cmark_mem *mem, cmark_node *root, int options, int width, void (*outc)(cmark_renderer *, cmark_node *, cmark_escaping, int32_t, @@ -169,10 +214,14 @@ char *cmark_render(cmark_mem *mem, cmark_node *root, int options, int width, char *result; cmark_iter *iter = cmark_iter_new(root); + bool outc_verbatim = outc == NULL; + if (outc_verbatim) + outc = S_outc_verbatim; + cmark_renderer renderer = {mem, &buf, &pref, 0, width, 0, 0, true, true, false, false, outc, S_cr, S_blankline, S_out, - 0}; + 0, outc_verbatim}; while ((ev_type = cmark_iter_next(iter)) != CMARK_EVENT_DONE) { cur = cmark_iter_get_node(iter); diff --git a/src/render.h b/src/render.h index 4a68d1e07..655a097b5 100644 --- a/src/render.h +++ b/src/render.h @@ -28,6 +28,7 @@ struct cmark_renderer { void (*blankline)(struct cmark_renderer *); void (*out)(struct cmark_renderer *, cmark_node *, const char *, bool, cmark_escaping); unsigned int footnote_ix; + bool outc_verbatim; }; typedef struct cmark_renderer cmark_renderer; @@ -47,6 +48,8 @@ void cmark_render_ascii(cmark_renderer *renderer, const char *s); void cmark_render_code_point(cmark_renderer *renderer, uint32_t c); +// A NULL outc writes every code point unchanged, which lets S_out copy runs. + char *cmark_render(cmark_mem *mem, cmark_node *root, int options, int width, void (*outc)(cmark_renderer *, cmark_node *, cmark_escaping, int32_t, From 23f5ae2bfa456cf49ac7e232a90648c1e83387f3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Joubert?= Date: Tue, 27 Oct 2026 12:17:29 +0300 Subject: [PATCH 2/2] Keep U+FFFE and U+FFFF out of verbatim runs cmark_utf8proc_encode_char writes these two code points as a single byte, so copying their original three bytes changed the output. Co-Authored-By: Claude Opus 5.5 --- src/render.c | 42 ++++++++++++++++++++++-------------------- src/render.h | 2 -- 2 files changed, 22 insertions(+), 22 deletions(-) diff --git a/src/render.c b/src/render.c index 28483a762..9d97deffc 100644 --- a/src/render.c +++ b/src/render.c @@ -19,6 +19,12 @@ static CMARK_INLINE void S_blankline(cmark_renderer *renderer) { } } +static void S_outc_verbatim(cmark_renderer *renderer, cmark_node *node, + cmark_escaping escape, int32_t c, + unsigned char nextc) { + cmark_render_code_point(renderer, c); +} + static void S_out(cmark_renderer *renderer, cmark_node *node, const char *source, bool wrap, cmark_escaping escape) { @@ -59,8 +65,10 @@ static void S_out(cmark_renderer *renderer, cmark_node *node, // Characters the renderer writes back unchanged are copied in runs rather // than decoded and re-encoded one by one. Spaces (when wrapping), newlines - // (in LITERAL) and invalid UTF-8 still go through the loop below. - bool verbatim = !ext && (escape == LITERAL || renderer->outc_verbatim); + // (in LITERAL), U+FFFE, U+FFFF and invalid UTF-8 still go through the loop + // below. + bool verbatim = + !ext && (escape == LITERAL || renderer->outc == S_outc_verbatim); while (i < length) { if (renderer->begin_line) { @@ -70,7 +78,7 @@ static void S_out(cmark_renderer *renderer, cmark_node *node, renderer->column = renderer->prefix->size; } - // A run cannot cross a line-break opportunity: last_breakable stays 0. + // With no pending break point, the width check below cannot fire. if (verbatim && (renderer->width == 0 || renderer->last_breakable == 0)) { int run_end = i; int code_points = 0; @@ -78,19 +86,20 @@ static void S_out(cmark_renderer *renderer, cmark_node *node, while (run_end < length) { unsigned char byte = (unsigned char)source[run_end]; if (byte < 0x80) { - if ((byte == ' ' && wrap) || (byte == '\n' && escape == LITERAL)) - break; - all_digits = all_digits && cmark_isdigit((char)byte); - run_end += 1; + c = byte; + len = 1; } else { len = cmark_utf8proc_iterate((const uint8_t *)source + run_end, length - run_end, &c); - if (len == -1) + // cmark_utf8proc_encode_char writes U+FFFE and U+FFFF as one byte. + if (len == -1 || c == 0xFFFE || c == 0xFFFF) break; - // (char)c truncates, as the per-character path below does. - all_digits = all_digits && cmark_isdigit((char)c); - run_end += len; } + if ((c == ' ' && wrap) || (c == '\n' && escape == LITERAL)) + break; + // (char)c truncates, as the per-character path below does. + all_digits = all_digits && cmark_isdigit((char)c); + run_end += len; code_points += 1; } if (run_end > i) { @@ -194,12 +203,6 @@ void cmark_render_code_point(cmark_renderer *renderer, uint32_t c) { renderer->column += 1; } -static void S_outc_verbatim(cmark_renderer *renderer, cmark_node *node, - cmark_escaping escape, int32_t c, - unsigned char nextc) { - cmark_render_code_point(renderer, c); -} - char *cmark_render(cmark_mem *mem, cmark_node *root, int options, int width, void (*outc)(cmark_renderer *, cmark_node *, cmark_escaping, int32_t, @@ -214,14 +217,13 @@ char *cmark_render(cmark_mem *mem, cmark_node *root, int options, int width, char *result; cmark_iter *iter = cmark_iter_new(root); - bool outc_verbatim = outc == NULL; - if (outc_verbatim) + if (outc == NULL) outc = S_outc_verbatim; cmark_renderer renderer = {mem, &buf, &pref, 0, width, 0, 0, true, true, false, false, outc, S_cr, S_blankline, S_out, - 0, outc_verbatim}; + 0}; while ((ev_type = cmark_iter_next(iter)) != CMARK_EVENT_DONE) { cur = cmark_iter_get_node(iter); diff --git a/src/render.h b/src/render.h index 655a097b5..0c6c079e1 100644 --- a/src/render.h +++ b/src/render.h @@ -28,7 +28,6 @@ struct cmark_renderer { void (*blankline)(struct cmark_renderer *); void (*out)(struct cmark_renderer *, cmark_node *, const char *, bool, cmark_escaping); unsigned int footnote_ix; - bool outc_verbatim; }; typedef struct cmark_renderer cmark_renderer; @@ -49,7 +48,6 @@ void cmark_render_ascii(cmark_renderer *renderer, const char *s); void cmark_render_code_point(cmark_renderer *renderer, uint32_t c); // A NULL outc writes every code point unchanged, which lets S_out copy runs. - char *cmark_render(cmark_mem *mem, cmark_node *root, int options, int width, void (*outc)(cmark_renderer *, cmark_node *, cmark_escaping, int32_t,