Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
47 changes: 41 additions & 6 deletions apps/server/src/provider/documentTextWorker.ts
Original file line number Diff line number Diff line change
Expand Up @@ -416,10 +416,35 @@ function* notebook(source: string): Generator<string, void> {
}
}

// Per-byte decoding only supports the single-byte \fcharset values.
const FONT_CODE_PAGES = new Map([
[77, "macintosh"],
[161, "windows-1253"],
[162, "windows-1254"],
[163, "windows-1258"],
[177, "windows-1255"],
[178, "windows-1256"],
[186, "windows-1257"],
[204, "windows-1251"],
[222, "windows-874"],
[238, "windows-1250"],
]);

function* rtf(source: string): Generator<string, void> {
if (!source.startsWith("{\\rtf")) throw new Error("Invalid RTF document");
let state = { skip: false, deleted: false, uc: 1, decoder: new TextDecoder("windows-1252") };
let state = {
skip: false,
deleted: false,
uc: 1,
font: 0,
fontTable: false,
decoder: new TextDecoder("windows-1252"),
};
const stack: Array<typeof state> = [];
// A font's \fcharset, not the document code page, encodes the bytes written in that font.
const fonts = new Map<number, typeof state.decoder>();
let defaultFont = 0;
const decoder = () => fonts.get(state.font) ?? state.decoder;
const hidden = () => state.skip || state.deleted;
const symbols = new Map([
["endash", "–"],
Expand Down Expand Up @@ -454,7 +479,7 @@ function* rtf(source: string): Generator<string, void> {
const word = /^([a-z]+)(-?\d+)? ?/i.exec(rest);
if (hex) {
if (fallback) fallback--;
else if (!hidden()) yield state.decoder.decode(Uint8Array.of(Number.parseInt(hex[1]!, 16)));
else if (!hidden()) yield decoder().decode(Uint8Array.of(Number.parseInt(hex[1]!, 16)));
i += 3;
} else if (word) {
i += word[0].length;
Expand Down Expand Up @@ -489,8 +514,20 @@ function* rtf(source: string): Generator<string, void> {
].includes(control)
)
state.skip = true;
if (control === "fonttbl") state.fontTable = true;
else if (control === "deff") state.font = defaultFont = value;
else if (control === "f") {
state.font = value;
if (state.fontTable) fonts.delete(value);
} else if (control === "fcharset" && state.fontTable) {
const codePage = FONT_CODE_PAGES.get(value);
if (codePage) fonts.set(state.font, new TextDecoder(codePage));
}
if (control === "deleted") state.deleted = value !== 0;
else if (control === "plain") state.deleted = false;
else if (control === "plain") {
state.deleted = false;
state.font = defaultFont;
}
if (control === "mac" || (control === "ansicpg" && value === 10000))
state.decoder = new TextDecoder("macintosh");
else if (control === "ansicpg") {
Expand Down Expand Up @@ -524,9 +561,7 @@ function* rtf(source: string): Generator<string, void> {
} else if (char !== "\n" && char !== "\r") {
if (fallback) fallback--;
else if (!hidden())
yield char.charCodeAt(0) < 128
? char
: state.decoder.decode(Uint8Array.of(char.charCodeAt(0)));
yield char.charCodeAt(0) < 128 ? char : decoder().decode(Uint8Array.of(char.charCodeAt(0)));
}
}
}
Expand Down
11 changes: 11 additions & 0 deletions apps/server/src/provider/userInputAttachments.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -344,6 +344,17 @@ describe("question answer paths", () => {
String.raw`{\rtf1\mac\ansicpg10000 Caf\'8e Cr\'8fme}`,
"Café Crème",
],
[
"font character set",
String.raw`{\rtf1\ansi\ansicpg1252\deff1{\fonttbl\f0\fnil\fcharset178 GeezaPro;\f1\fnil\fcharset0 LucidaGrande;}
\f0 \'e3\'e1\'c7\'cd\'d9\'c9\f1 : caf\'e9, \f0\plain\'c7a}`,
"ملاحظة: café, Ça",
],
[
"font character set outside the font table",
String.raw`{\rtf1\ansi{\fonttbl{\f0\fcharset178 A;}{\f0\fcharset0 B;}{\f1\fcharset0 C;}{\f2\fcharset178 D;}{\f2 E;}}{\stylesheet{\s0\f1\fcharset178 N;}}\f0 \'e3\f1\'e3\f2\'e3}`,
"ããã",
],
["destination symbol", String.raw`{\rtf1\ansi\u945\*Z}`, "αZ"],
["multiple units", String.raw`{\rtf1\ansi\uc4\u945?\tab\'3f\~Z}`, "αZ"],
["binary unit", String.raw`{\rtf1\ansi\u945\bin3 abcZ}`, "αZ"],
Expand Down
Loading