diff --git a/apps/server/src/provider/documentTextWorker.ts b/apps/server/src/provider/documentTextWorker.ts index 9cec14dadd67..b49b9a42378f 100644 --- a/apps/server/src/provider/documentTextWorker.ts +++ b/apps/server/src/provider/documentTextWorker.ts @@ -416,10 +416,35 @@ function* notebook(source: string): Generator { } } +// Per-byte decoding only supports the single-byte \fcharset values. +const FONT_CODE_PAGES = new Map([ + [77, "macintosh"], + [161, "windows-1253"], + [162, "windows-1254"], + [163, "windows-1258"], + [177, "windows-1255"], + [178, "windows-1256"], + [186, "windows-1257"], + [204, "windows-1251"], + [222, "windows-874"], + [238, "windows-1250"], +]); + function* rtf(source: string): Generator { if (!source.startsWith("{\\rtf")) throw new Error("Invalid RTF document"); - let state = { skip: false, deleted: false, uc: 1, decoder: new TextDecoder("windows-1252") }; + let state = { + skip: false, + deleted: false, + uc: 1, + font: 0, + fontTable: false, + decoder: new TextDecoder("windows-1252"), + }; const stack: Array = []; + // A font's \fcharset, not the document code page, encodes the bytes written in that font. + const fonts = new Map(); + let defaultFont = 0; + const decoder = () => fonts.get(state.font) ?? state.decoder; const hidden = () => state.skip || state.deleted; const symbols = new Map([ ["endash", "–"], @@ -454,7 +479,7 @@ function* rtf(source: string): Generator { const word = /^([a-z]+)(-?\d+)? ?/i.exec(rest); if (hex) { if (fallback) fallback--; - else if (!hidden()) yield state.decoder.decode(Uint8Array.of(Number.parseInt(hex[1]!, 16))); + else if (!hidden()) yield decoder().decode(Uint8Array.of(Number.parseInt(hex[1]!, 16))); i += 3; } else if (word) { i += word[0].length; @@ -489,8 +514,20 @@ function* rtf(source: string): Generator { ].includes(control) ) state.skip = true; + if (control === "fonttbl") state.fontTable = true; + else if (control === "deff") state.font = defaultFont = value; + else if (control === "f") { + state.font = value; + if (state.fontTable) fonts.delete(value); + } else if (control === "fcharset" && state.fontTable) { + const codePage = FONT_CODE_PAGES.get(value); + if (codePage) fonts.set(state.font, new TextDecoder(codePage)); + } if (control === "deleted") state.deleted = value !== 0; - else if (control === "plain") state.deleted = false; + else if (control === "plain") { + state.deleted = false; + state.font = defaultFont; + } if (control === "mac" || (control === "ansicpg" && value === 10000)) state.decoder = new TextDecoder("macintosh"); else if (control === "ansicpg") { @@ -524,9 +561,7 @@ function* rtf(source: string): Generator { } else if (char !== "\n" && char !== "\r") { if (fallback) fallback--; else if (!hidden()) - yield char.charCodeAt(0) < 128 - ? char - : state.decoder.decode(Uint8Array.of(char.charCodeAt(0))); + yield char.charCodeAt(0) < 128 ? char : decoder().decode(Uint8Array.of(char.charCodeAt(0))); } } } diff --git a/apps/server/src/provider/userInputAttachments.test.ts b/apps/server/src/provider/userInputAttachments.test.ts index 7bf24acb0b40..26752022aab1 100644 --- a/apps/server/src/provider/userInputAttachments.test.ts +++ b/apps/server/src/provider/userInputAttachments.test.ts @@ -344,6 +344,17 @@ describe("question answer paths", () => { String.raw`{\rtf1\mac\ansicpg10000 Caf\'8e Cr\'8fme}`, "Café Crème", ], + [ + "font character set", + String.raw`{\rtf1\ansi\ansicpg1252\deff1{\fonttbl\f0\fnil\fcharset178 GeezaPro;\f1\fnil\fcharset0 LucidaGrande;} +\f0 \'e3\'e1\'c7\'cd\'d9\'c9\f1 : caf\'e9, \f0\plain\'c7a}`, + "ملاحظة: café, Ça", + ], + [ + "font character set outside the font table", + String.raw`{\rtf1\ansi{\fonttbl{\f0\fcharset178 A;}{\f0\fcharset0 B;}{\f1\fcharset0 C;}{\f2\fcharset178 D;}{\f2 E;}}{\stylesheet{\s0\f1\fcharset178 N;}}\f0 \'e3\f1\'e3\f2\'e3}`, + "ããã", + ], ["destination symbol", String.raw`{\rtf1\ansi\u945\*Z}`, "αZ"], ["multiple units", String.raw`{\rtf1\ansi\uc4\u945?\tab\'3f\~Z}`, "αZ"], ["binary unit", String.raw`{\rtf1\ansi\u945\bin3 abcZ}`, "αZ"],