From 9df395ceeba4fba31aea4fe949bb4dc6a6b5f68f Mon Sep 17 00:00:00 2001 From: Quin Gillespie Date: Sat, 31 Oct 2026 10:37:32 -0600 Subject: [PATCH] fix(lexer): close the group after an escaped backslash The scanner treated any "{", "}", "\" or newline right after a backslash as escaped, so in "\}" the brace was taken as escaped too: the escaped backslash escaped the next character as well. The group never closed, and every token after it was off by one bracket. Word writes exactly this inside field instructions such as {\fldinst SYMBOL 183 \f "Symbol"}, and documents containing them lost everything that followed. --- src/lexer.rs | 25 ++++++++++++++++++++++--- 1 file changed, 22 insertions(+), 3 deletions(-) diff --git a/src/lexer.rs b/src/lexer.rs index 59d68e6..7880fbd 100644 --- a/src/lexer.rs +++ b/src/lexer.rs @@ -45,13 +45,14 @@ impl Lexer { let mut tokens: Vec = vec![]; let mut slice_start_index = 0; - let mut previous_char = ' '; + // Whether the previous char is a backslash that still escapes this one + let mut escaping = false; for (current_index, c) in src.char_indices() { match c { // TODO: Handle char over code 127 for escaped chars // Handle Escaped chars : "\" + any charcode below 127 - '{' | '}' | '\\' | '\n' if previous_char == '\\' => {} + '{' | '}' | '\\' | '\n' if escaping => {} '{' | '}' | '\\' | '\n' => { // End of slice chars if slice_start_index < current_index { @@ -66,7 +67,8 @@ impl Lexer { // Others chars _ => {} } - previous_char = c; + // An escaped backslash escapes nothing itself, so the "}" in "\\}" still closes a group + escaping = c == '\\' && !escaping; } // Manage last token (should always be "}") if slice_start_index < src.len() { @@ -190,6 +192,23 @@ pub(crate) mod tests { ); } + #[test] + fn scan_escaped_backslash_before_closing_bracket() { + let tokens = Lexer::scan(r#"{\f0 C:\\}after}"#); + assert_eq!( + tokens.unwrap(), + vec![ + OpeningBracket, + ControlSymbol((FontNumber, Value(0))), + PlainText("C:"), + PlainText("\\"), + ClosingBracket, + PlainText("after"), + ClosingBracket + ], + ); + } + #[test] fn scan_escaped_text() { let tokens = Lexer::scan(