lexer: filter only newly read tokens, not the whole buffer

Author: gf712

src/lexer/Lexer.cpp

@@ -78,17 +78,23 @@ bool Lexer::read_more_tokens()
 {
 	const std::size_t token_size = m_tokens_to_emit.size();
 	while (token_size >= m_tokens_to_emit.size()) {
+		const std::size_t already_filtered = m_tokens_to_emit.size();
 		bool read_tokens = read_more_tokens_loop();
 		if (read_tokens) {
-			std::erase_if(m_tokens_to_emit, [this](Token &token) {
-				if (m_ignore_comments && token.token_type() == Token::TokenType::COMMENT) {
-					return true;
-				}
-				if (m_ignore_nl_token && token.token_type() == Token::TokenType::NL) {
-					return true;
-				}
-				return false;
-			});
+			const auto first = m_tokens_to_emit.begin() + already_filtered;
+			m_tokens_to_emit.erase(
+				std::remove_if(first,
+					m_tokens_to_emit.end(),
+					[this](const Token &token) {
+						if (m_ignore_comments && token.token_type() == Token::TokenType::COMMENT) {
+							return true;
+						}
+						if (m_ignore_nl_token && token.token_type() == Token::TokenType::NL) {
+							return true;
+						}
+						return false;
+					}),
+				m_tokens_to_emit.end());
 		} else {
 			return false;
 		}
@@ -974,4 +980,4 @@ Token Lexer::pop_front()
 	auto &result = m_tokens_to_emit.front();
 	m_tokens_to_emit.pop_front();
 	return result;
-}
+}

src/lexer/Lexer_tests.cpp

@@ -609,6 +609,24 @@ TEST(Lexer, IgnoreCommentsAndNL)
 	assert_generates_tokens_without_comment_tokens(program, expected_tokens);
 }
 
+TEST(Lexer, IgnoreCommentsAndNLLateInTheProgram)
+{
+	constexpr std::size_t line_count = 200;
+	std::string program;
+	std::vector<Token::TokenType> expected_tokens;
+	for (std::size_t i = 0; i < line_count; ++i) {
+		program += std::format("# leading comment {}\n", i);
+		program += "\n";
+		program += std::format("a{} = {} # trailing comment\n", i, i);
+		expected_tokens.push_back(Token::TokenType::NAME);
+		expected_tokens.push_back(Token::TokenType::EQUAL);
+		expected_tokens.push_back(Token::TokenType::NUMBER);
+		expected_tokens.push_back(Token::TokenType::NEWLINE);
+	}
+	expected_tokens.push_back(Token::TokenType::ENDMARKER);
+	assert_generates_tokens_without_comment_tokens(program, expected_tokens);
+}
+
 TEST(Lexer, Ellipsis)
 {
 	constexpr std::string_view program = "...\n";