From 2116dfc1d0177eb22334e870574bd6fd740f30af Mon Sep 17 00:00:00 2001 From: Michel Liao Date: Sat, 21 Nov 2026 13:23:51 -0500 Subject: [PATCH 1/2] fix(server): RTF attachments no longer garble text set in a font's own character set MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit RTF writes non-ASCII text as \'hh bytes in the character set of the current font (\fcharsetN in the font table), not the document code page. The attachment text extractor decoded every byte with the document code page, so a TextEdit document with Arabic in Geeza Pro (\fcharset178 under \ansicpg1252) reached the agent as "ãáÇÍÙÉ" instead of "ملاحظة". The web preview already reads these fonts correctly. Track the current font (\f, \deff, \plain) and decode bytes with the font's code page when its font table entry names a single-byte one. Multi-byte font character sets are unchanged. Claude Fable 5.2 in Claude Code, running in T3 Code. Co-Authored-By: Claude Fable 5.2 --- .../server/src/provider/documentTextWorker.ts | 46 ++++++++++++++++--- .../src/provider/userInputAttachments.test.ts | 11 +++++ 2 files changed, 51 insertions(+), 6 deletions(-) diff --git a/apps/server/src/provider/documentTextWorker.ts b/apps/server/src/provider/documentTextWorker.ts index 9cec14dadd67..e7647bcac6e6 100644 --- a/apps/server/src/provider/documentTextWorker.ts +++ b/apps/server/src/provider/documentTextWorker.ts @@ -416,10 +416,35 @@ function* notebook(source: string): Generator { } } +// Per-byte decoding only supports the single-byte \fcharset values. +const FONT_CODE_PAGES = new Map([ + [77, "macintosh"], + [161, "windows-1253"], + [162, "windows-1254"], + [163, "windows-1258"], + [177, "windows-1255"], + [178, "windows-1256"], + [186, "windows-1257"], + [204, "windows-1251"], + [222, "windows-874"], + [238, "windows-1250"], +]); + function* rtf(source: string): Generator { if (!source.startsWith("{\\rtf")) throw new Error("Invalid RTF document"); - let state = { skip: false, deleted: false, uc: 1, decoder: new TextDecoder("windows-1252") }; + let state = { + skip: false, + deleted: false, + uc: 1, + font: 0, + fontTable: false, + decoder: new TextDecoder("windows-1252"), + }; const stack: Array = []; + // A font's \fcharset, not the document code page, encodes the bytes written in that font. + const fonts = new Map(); + let defaultFont = 0; + const decoder = () => fonts.get(state.font) ?? state.decoder; const hidden = () => state.skip || state.deleted; const symbols = new Map([ ["endash", "–"], @@ -454,7 +479,7 @@ function* rtf(source: string): Generator { const word = /^([a-z]+)(-?\d+)? ?/i.exec(rest); if (hex) { if (fallback) fallback--; - else if (!hidden()) yield state.decoder.decode(Uint8Array.of(Number.parseInt(hex[1]!, 16))); + else if (!hidden()) yield decoder().decode(Uint8Array.of(Number.parseInt(hex[1]!, 16))); i += 3; } else if (word) { i += word[0].length; @@ -489,8 +514,19 @@ function* rtf(source: string): Generator { ].includes(control) ) state.skip = true; + if (control === "fonttbl") state.fontTable = true; + else if (control === "deff") state.font = defaultFont = value; + else if (control === "f") state.font = value; + else if (control === "fcharset" && state.fontTable) { + const codePage = FONT_CODE_PAGES.get(value); + if (codePage) fonts.set(state.font, new TextDecoder(codePage)); + else fonts.delete(state.font); + } if (control === "deleted") state.deleted = value !== 0; - else if (control === "plain") state.deleted = false; + else if (control === "plain") { + state.deleted = false; + state.font = defaultFont; + } if (control === "mac" || (control === "ansicpg" && value === 10000)) state.decoder = new TextDecoder("macintosh"); else if (control === "ansicpg") { @@ -524,9 +560,7 @@ function* rtf(source: string): Generator { } else if (char !== "\n" && char !== "\r") { if (fallback) fallback--; else if (!hidden()) - yield char.charCodeAt(0) < 128 - ? char - : state.decoder.decode(Uint8Array.of(char.charCodeAt(0))); + yield char.charCodeAt(0) < 128 ? char : decoder().decode(Uint8Array.of(char.charCodeAt(0))); } } } diff --git a/apps/server/src/provider/userInputAttachments.test.ts b/apps/server/src/provider/userInputAttachments.test.ts index 7bf24acb0b40..71ebf572ac74 100644 --- a/apps/server/src/provider/userInputAttachments.test.ts +++ b/apps/server/src/provider/userInputAttachments.test.ts @@ -344,6 +344,17 @@ describe("question answer paths", () => { String.raw`{\rtf1\mac\ansicpg10000 Caf\'8e Cr\'8fme}`, "Café Crème", ], + [ + "font character set", + String.raw`{\rtf1\ansi\ansicpg1252\deff1{\fonttbl\f0\fnil\fcharset178 GeezaPro;\f1\fnil\fcharset0 LucidaGrande;} +\f0 \'e3\'e1\'c7\'cd\'d9\'c9\f1 : caf\'e9, \f0\plain\'c7a}`, + "ملاحظة: café, Ça", + ], + [ + "font character set outside the font table", + String.raw`{\rtf1\ansi{\fonttbl{\f0\fcharset178 A;}{\f0\fcharset0 B;}{\f1\fcharset0 C;}}{\stylesheet{\s0\f1\fcharset178 N;}}\f0 \'e3\f1\'e3}`, + "ãã", + ], ["destination symbol", String.raw`{\rtf1\ansi\u945\*Z}`, "αZ"], ["multiple units", String.raw`{\rtf1\ansi\uc4\u945?\tab\'3f\~Z}`, "αZ"], ["binary unit", String.raw`{\rtf1\ansi\u945\bin3 abcZ}`, "αZ"], From 0909a2f5c20e3314078846604b1a19d33a99269c Mon Sep 17 00:00:00 2001 From: Michel Liao Date: Sat, 21 Nov 2026 13:51:47 -0500 Subject: [PATCH 2/2] fix(server): a re-declared RTF font drops its earlier character set A font table entry may omit \fcharset. When a font number was declared twice and the second entry had none, the first entry's character set stayed in effect. Starting a font table entry now clears that font's mapping, so only the entry's own \fcharset applies. Claude Fable 5.2 in Claude Code, running in T3 Code. Co-Authored-By: Claude Fable 5.2 --- apps/server/src/provider/documentTextWorker.ts | 7 ++++--- apps/server/src/provider/userInputAttachments.test.ts | 4 ++-- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/apps/server/src/provider/documentTextWorker.ts b/apps/server/src/provider/documentTextWorker.ts index e7647bcac6e6..b49b9a42378f 100644 --- a/apps/server/src/provider/documentTextWorker.ts +++ b/apps/server/src/provider/documentTextWorker.ts @@ -516,11 +516,12 @@ function* rtf(source: string): Generator { state.skip = true; if (control === "fonttbl") state.fontTable = true; else if (control === "deff") state.font = defaultFont = value; - else if (control === "f") state.font = value; - else if (control === "fcharset" && state.fontTable) { + else if (control === "f") { + state.font = value; + if (state.fontTable) fonts.delete(value); + } else if (control === "fcharset" && state.fontTable) { const codePage = FONT_CODE_PAGES.get(value); if (codePage) fonts.set(state.font, new TextDecoder(codePage)); - else fonts.delete(state.font); } if (control === "deleted") state.deleted = value !== 0; else if (control === "plain") { diff --git a/apps/server/src/provider/userInputAttachments.test.ts b/apps/server/src/provider/userInputAttachments.test.ts index 71ebf572ac74..26752022aab1 100644 --- a/apps/server/src/provider/userInputAttachments.test.ts +++ b/apps/server/src/provider/userInputAttachments.test.ts @@ -352,8 +352,8 @@ describe("question answer paths", () => { ], [ "font character set outside the font table", - String.raw`{\rtf1\ansi{\fonttbl{\f0\fcharset178 A;}{\f0\fcharset0 B;}{\f1\fcharset0 C;}}{\stylesheet{\s0\f1\fcharset178 N;}}\f0 \'e3\f1\'e3}`, - "ãã", + String.raw`{\rtf1\ansi{\fonttbl{\f0\fcharset178 A;}{\f0\fcharset0 B;}{\f1\fcharset0 C;}{\f2\fcharset178 D;}{\f2 E;}}{\stylesheet{\s0\f1\fcharset178 N;}}\f0 \'e3\f1\'e3\f2\'e3}`, + "ããã", ], ["destination symbol", String.raw`{\rtf1\ansi\u945\*Z}`, "αZ"], ["multiple units", String.raw`{\rtf1\ansi\uc4\u945?\tab\'3f\~Z}`, "αZ"],